Streaming video .f4v files via Amazon S3 and CloudFront

The trick is to pre-pend the playpath with "mp4:" (not f4v: or flv:).

host50 $ ./rtmpdump -V -r rtmp://[cloudfront_streaming_distribution_host].cloudfront.net:1935/cfx/st --app "cfx/st" --playpath "mp4:myvideofile.f4v" 
RTMPDump v2.3
(c) 2010 Andrej Stepanchuk, Howard Chu, The Flvstreamer Team; license: GPL
DEBUG: Parsing...
DEBUG: Parsed protocol: 0
DEBUG: Parsed host    : [cloudfront_streaming_distribution_host].cloudfront.net
DEBUG: Parsed app     : cfx
WARNING: You haven't specified an output file (-o filename), using stdout
DEBUG: Protocol : RTMP
DEBUG: Hostname : [cloudfront_streaming_distribution_host].cloudfront.net
DEBUG: Port     : 1935
DEBUG: Playpath : mp4:myvideofile.f4v
DEBUG: tcUrl    : rtmp://[cloudfront_streaming_distribution_host].cloudfront.net:1935/cfx/st
DEBUG: app      : cfx/st
DEBUG: live     : no
DEBUG: timeout  : 30 sec
DEBUG: Setting buffer time to: 36000000ms
Connecting ...
DEBUG: RTMP_Connect1, ... connected, handshaking
DEBUG: HandShake: Type Answer   : 03
DEBUG: HandShake: Server Uptime : 1249013055
DEBUG: HandShake: FMS Version   : 3.5.7.1
WARNING: HandShake: client signature does not match!
DEBUG: HandShake: Handshaking finished....
DEBUG: RTMP_Connect1, handshaked
DEBUG: Invoking connect
INFO: Connected...
DEBUG: HandleServerBW: server BW = 1310720
DEBUG: HandleClientBW: client BW = 2500000 2
DEBUG: RTMP_ClientPacket, received: invoke 242 bytes
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:             fmsVer, STRING: FMS/3,5,7,7009>
DEBUG: Property: <Name:       capabilities, NUMBER: 127.00>
DEBUG: Property: <Name:               mode, NUMBER: 1.00>
DEBUG: (object end)
DEBUG: (object begin)
DEBUG: Property: <Name:              level, STRING: status>
DEBUG: Property: <Name:               code, STRING: NetConnection.Connect.Success>
DEBUG: Property: <Name:        description, STRING: Connection succeeded.>
DEBUG: Property: <Name:     objectEncoding, NUMBER: 0.00>
DEBUG: Property: <Name:               data, OBJECT>
DEBUG: (object begin)
DEBUG: Property: <Name:            version, STRING: 3,5,7,7009>
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object end)
DEBUG: HandleInvoke, server invoking <_result>
DEBUG: HandleInvoke, received result for method call <connect>
DEBUG: sending ctrl. type: 0x0003
DEBUG: Invoking createStream
DEBUG: RTMP_ClientPacket, received: invoke 21 bytes
DEBUG: (object begin)
DEBUG: Property: NULL
DEBUG: (object end)
DEBUG: HandleInvoke, server invoking <onBWDone>
DEBUG: Invoking _checkbw
DEBUG: RTMP_ClientPacket, received: invoke 29 bytes
DEBUG: (object begin)
DEBUG: Property: NULL
DEBUG: (object end)
DEBUG: HandleInvoke, server invoking <_result>
DEBUG: HandleInvoke, received result for method call <createStream>
DEBUG: SendPlay, seekTime=0, stopTime=0, sending play: mp4:myvideofile.f4v
DEBUG: Invoking play
DEBUG: sending ctrl. type: 0x0003
DEBUG: RTMP_ClientPacket, received: invoke 16419 bytes
DEBUG: (object begin)
DEBUG: Property: NULL
DEBUG: (object end)
DEBUG: HandleInvoke, server invoking <_onbwcheck>
DEBUG: Invoking _result
DEBUG: HandleChangeChunkSize, received: chunk size change to 4096
DEBUG: HandleCtrl, received ctrl. type: 4, len: 6
DEBUG: HandleCtrl, Stream IsRecorded 1
DEBUG: HandleCtrl, received ctrl. type: 0, len: 6
DEBUG: HandleCtrl, Stream Begin 1
DEBUG: RTMP_ClientPacket, received: invoke 164 bytes
DEBUG: (object begin)
DEBUG: Property: NULL
DEBUG: (object begin)
DEBUG: Property: <Name:              level, STRING: status>
DEBUG: Property: <Name:               code, STRING: NetStream.Play.Reset>
DEBUG: Property: <Name:        description, STRING: Playing and resetting myvideofile.f4v.>
DEBUG: Property: <Name:            details, STRING: myvideofile.f4v>
DEBUG: Property: <Name:           clientid, STRING: 0BiIux^w>
DEBUG: (object end)
DEBUG: (object end)
DEBUG: HandleInvoke, server invoking <onStatus>
DEBUG: HandleInvoke, onStatus: NetStream.Play.Reset
DEBUG: RTMP_ClientPacket, received: invoke 158 bytes
DEBUG: (object begin)
DEBUG: Property: NULL
DEBUG: (object begin)
DEBUG: Property: <Name:              level, STRING: status>
DEBUG: Property: <Name:               code, STRING: NetStream.Play.Start>
DEBUG: Property: <Name:        description, STRING: Started playing myvideofile.f4v.>
DEBUG: Property: <Name:            details, STRING: myvideofile.f4v>
DEBUG: Property: <Name:           clientid, STRING: 0BiIux^w>
DEBUG: (object end)
DEBUG: (object end)
DEBUG: HandleInvoke, server invoking <onStatus>
DEBUG: HandleInvoke, onStatus: NetStream.Play.Start
Starting download at: 5137.107 kB
DEBUG: RTMP_ClientPacket, received: notify 24 bytes
DEBUG: (object begin)
DEBUG: (object end)
DEBUG: ignoring too small audio packet: size: 0
DEBUG: RTMP_ClientPacket, received: notify 44 bytes
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:               code, STRING: NetStream.Data.Start>
DEBUG: (object end)
DEBUG: (object end)
DEBUG: RTMP_ClientPacket, received: notify 598 bytes
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:           duration, NUMBER: 89.67>
DEBUG: Property: <Name:       moovPosition, NUMBER: 36.00>
DEBUG: Property: <Name:              width, NUMBER: 960.00>
DEBUG: Property: <Name:             height, NUMBER: 540.00>
DEBUG: Property: <Name:       videocodecid, STRING: avc1>
DEBUG: Property: <Name:       audiocodecid, STRING: mp4a>
DEBUG: Property: <Name:         avcprofile, NUMBER: 77.00>
DEBUG: Property: <Name:           avclevel, NUMBER: 40.00>
DEBUG: Property: <Name:             aacaot, NUMBER: 2.00>
DEBUG: Property: <Name:     videoframerate, NUMBER: 29.97>
DEBUG: Property: <Name:    audiosamplerate, NUMBER: 44100.00>
DEBUG: Property: <Name:      audiochannels, NUMBER: 2.00>
DEBUG: Property: <Name:          trackinfo, OBJECT>
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:             length, NUMBER: 8066058.00>
DEBUG: Property: <Name:          timescale, NUMBER: 90000.00>
DEBUG: Property: <Name:           language, STRING: eng>
DEBUG: Property: <Name:  sampledescription, OBJECT>
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:         sampletype, STRING: avc1>
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object begin)
DEBUG: Property: <Name:             length, NUMBER: 3954688.00>
DEBUG: Property: <Name:          timescale, NUMBER: 44100.00>
DEBUG: Property: <Name:           language, STRING: eng>
DEBUG: Property: <Name:  sampledescription, OBJECT>
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:         sampletype, STRING: mp4a>
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object begin)
DEBUG: Property: <Name:             length, NUMBER: 8066058.00>
DEBUG: Property: <Name:          timescale, NUMBER: 90000.00>
DEBUG: Property: <Name:           language, STRING: eng>
DEBUG: Property: <Name:  sampledescription, OBJECT>
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:         sampletype, STRING: amf0>
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object end)
DEBUG: (object end)
INFO: Metadata:
INFO:   duration              89.67
INFO:   moovPosition          36.00
INFO:   width                 960.00
INFO:   height                540.00
INFO:   videocodecid          avc1
INFO:   audiocodecid          mp4a
INFO:   avcprofile            77.00
INFO:   avclevel              40.00
INFO:   aacaot                2.00
INFO:   videoframerate        29.97
INFO:   audiosamplerate       44100.00
INFO:   audiochannels         2.00
INFO: trackinfo:
INFO:   length                8066058.00
INFO:   timescale             90000.00
INFO:   language              eng
INFO: sampledescription:
INFO:   sampletype            avc1
INFO:   length                3954688.00
INFO:   timescale             44100.00
INFO:   language              eng
INFO: sampledescription:
INFO:   sampletype            mp4a
INFO:   length                8066058.00
INFO:   timescale             90000.00
INFO:   language              eng
INFO: sampledescription:
INFO:   sampletype            amf0
DEBUG: RTMP_ClientPacket, received: notify 4862 bytes
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:               data, STRING: <?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>
<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.0-c060 61.134777, 2010/02/12-17:32:00        ">
 <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
  <rdf:Descript>
DEBUG: (object end)
DEBUG: (object end)
FLV V
onMetaDaduration@Vk33333
                        moovPosition@Bwidth@?height@??
                                                      videocodecidavc1
                                                                      audiocodecidmp4a
audiochannels@cltrackinfoaot@videoframerate@=?S?Uk(audiosamplerate@刀
lengthA^?? timescale@?languageengsampledescription

sampletypeavc1  lengthAN, timescale@ languageengsampledescription

sampletypemp4a  lengthA^?? timescale@?languageengsampledescription

5137.777 kB / 0.00 sec (0.0%)sampletypeamf0   a -M@(gM@(?R?"??*>?? ?-h?sR8 K!Be??@v???E?Yh ?Df?>?j??u`|Qa?? (?) ???2|p

DEBUG: RTMP_ClientPacket, received: notify 224 bytes
DEBUG: (object begin)
DEBUG: (object begin)
DEBUG: Property: <Name:           duration, NUMBER: 89.62>
DEBUG: Property: <Name:              width, NUMBER: 960.00>
DEBUG: Property: <Name:             height, NUMBER: 540.00>
DEBUG: Property: <Name:      videodatarate, NUMBER: 5120.00>
DEBUG: Property: <Name:          framerate, NUMBER: 29.97>
DEBUG: Property: <Name:       videocodecid, NUMBER: 7.00>
DEBUG: Property: <Name:      audiodatarate, NUMBER: 128.00>
DEBUG: Property: <Name:         audiodelay, NUMBER: 0.04>
DEBUG: Property: <Name:       audiocodecid, NUMBER: 10.00>
DEBUG: Property: <Name:       canSeekToEnd, BOOLEAN: TRUE>
DEBUG: (object end)
DEBUG: (object end)
INFO: Metadata:
INFO:   duration              89.62
INFO:   width                 960.00
INFO:   height                540.00
INFO:   videodatarate         5120.00
INFO:   framerate             29.97
INFO:   videocodecid          7.00
INFO:   audiodatarate         128.00
INFO:   audiodelay            0.04
INFO:   audiocodecid          10.00
INFO:   canSeekToEnd          TRUE
v?!??7????
onMetaData
videodatarate@???;dZ@?heframerate@=?S
audiodatarate@`                      videocodecid@
audiodelay??n??O?;
                  audiocodecid@$
                                canSeekToEnd 

[lots of binary data here]

Reference:
https://forums.aws.amazon.com/thread.jspa?threadID=41165
http://aws.amazon.com/articles/4112
http://www.bucketexplorer.com/documentation/amazon-s3--how-to-stream-flv-video-from-amazons3.html
https://forums.aws.amazon.com/thread.jspa?threadID=40374

Download rtmpdump for Mac/Snow Leopard:
http://trick77.com/downloads/

Source:
http://trick77.com/2010/07/31/capture-flash-flv-streams-rtmpdump-mac-os-x/
http://www.videohelp.com/tools/RTMPDump

log file histograms

this one will show how many user queries we got per hour in july (excluding user 4890)

for i in 07; do for j in 01 02 03 04 05 06 07 08 09 10; do for k in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23; do echo 2011-${i}-${j} ${k} $( grep -i "${i}-${j} ${k}:" webapp.log | grep -v "User 4890" | grep -i "ran query" | wc -l); done; done; done;

.. and this one shows the distribution by hour in day for all of july (aggregate for all days)

# all of july by time of day
for i in 07; do for k in 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 18 19 20 21 22 23; do echo ${k} $( grep -i " ${k}:" webapp.log | grep -v "User 4890" | grep -i "ran query" | wc -l); done; done;
00 116
01 75
02 58
03 61
04 41
05 85

awk queries of tomcat and solr log files

tomcat log:
awk '/ran query/ {print $2 " " $18}' webapp.log

result:
00:00:08,878 10114ms
00:00:25,527 10103ms
00:00:32,089 54ms
00:00:38,661 49ms
00:00:44,874 58ms

solr log - query for a substring:
awk '/QTime=/ {print substr($5,index($5,"status:")+8,13) " " $8}' catalina.2012-02-03.log

result:
silverton)+OR QTime=47545
done)+OR+meta QTime=64246
facebook)+OR+ QTime=15506
google)+OR+me QTime=32403
mashable)+OR+ QTime=26
{facet.limit= QTime=1
{wt=javabin&r QTime=6
california)+O QTime=32
{facet.limit= QTime=39
{wt=javabin&r QTime=7

reference:
http://star-www.rl.ac.uk/docs/sc4.htx/node38.html

Using rsync to deploy from staging server to production

cat rsync_a_to_b.sh
# script that synchronizes files from location A to location B via a local temp directory
USER_A=ec2-user
HOST_A="$1"
PATH_A="$2"
USER_B=ec2-user
HOST_B="$3"
PATH_B="$4"

if [ -z $HOST_A ]; then # -n tests to see if the argument is non empty
echo "Missing source host argument, usage: [cmd] sourceHost sourcePath targetHost targetPath"
exit 1
fi

if [ -z $PATH_A ]; then # -n tests to see if the argument is non empty
echo "Missing source path argument, usage: [cmd] sourceHost sourcePath targetHost targetPath"
exit 1
fi

if [ -z $HOST_B ]; then # -n tests to see if the argument is non empty
echo "Missing target host argument, usage: [cmd] sourceHost sourcePath targetHost targetPath"
exit 1
fi

if [ -z $PATH_B ]; then # -n tests to see if the argument is non empty
echo "Missing target path argument, usage: [cmd] sourceHost sourcePath targetHost targetPath"
exit 1
fi

LOCAL_DIR=/tmp/rsync/${HOST_A}${PATH_A}

echo STEP 1: Syncing from ${HOST_A}:${PATH_A} to localhost:${LOCAL_DIR}..
mkdir -p ${LOCAL_DIR}

rsync -e ssh -rcav --delete-after \
${USER_A}@${HOST_A}:${PATH_A} \
--exclude '.svn/**' \
${LOCAL_DIR}

echo STEP 2: Syncing from localhost:${LOCAL_DIR} to ${HOST_B}:${PATH_B}..

rsync -e ssh -rcav --delete-after \
$LOCAL_DIR/* \
${USER_B}@${HOST_B}:${PATH_B}

echo DONE.

------------------------------

Then write a script to use the above function for every directory, like this:

cat push_stage_to_prod.sh
BASE=./

SOURCE_HOST=[mystagehost]
TARGET_HOST=[myprodhost]

echo Pushing entire application codebase from ${SOURCE_HOST} to ${TARGET_HOST}...

$BASE/rsync_a_to_b.sh $SOURCE_HOST \
'/my/app/tomcat/webapps/ROOT/WEB-INF/*.xml' \
$TARGET_HOST \
/my/app/tomcat/webapps/ROOT/WEB-INF/

$BASE/rsync_a_to_b.sh $SOURCE_HOST \
'/my/app/tomcat/webapps/ROOT/WEB-INF/classes/*' \
$TARGET_HOST \
/my/app/tomcat/webapps/ROOT/WEB-INF/classes/

$BASE/rsync_a_to_b.sh $SOURCE_HOST \
'/my/app/tomcat/webapps/ROOT/WEB-INF/lib/*' \
$TARGET_HOST \
/my/app/tomcat/webapps/ROOT/WEB-INF/lib/

$BASE/rsync_a_to_b.sh $SOURCE_HOST \
'/my/app/lib/m2reposymlink/*' \
$TARGET_HOST \
/my/app/lib/m2reposymlink/

$BASE/rsync_a_to_b.sh $SOURCE_HOST \
'/my/app/webcode/*' \
$TARGET_HOST \
/my/app/webcode/

File deployment script that uses rsync

cat function_rsync.sh
LOCAL_SYNC_PATH="$1"
REMOTE_USER="$2"
REMOTE_HOST="$3"
REMOTE_PATH="$4"

echo user=$REMOTE_USER
echo ..syncing directory "$LOCAL_SYNC_PATH" to server ${REMOTE_HOST}..

rsync -e ssh -rcav --delete-after \
$LOCAL_SYNC_PATH \
${REMOTE_USER}@${REMOTE_HOST}:${REMOTE_PATH}

Local subversion backup

# Settings
LOCALREPO='/root/backup/[myapp]/svn_repo'
REMOTEHOST='[myhost]'
REMOTEREPO='/var/www/svn/repos/[myapp]'
REMOTEUSER='[myuser]'

### WARNING: Danger Will Robinson ###
### DO NOT EDIT BELOW THIS LINE ###

LOCALVER=`/usr/bin/svnlook youngest ${LOCALREPO}`
REMOTEVER=`/usr/bin/ssh ${REMOTEUSER}@${REMOTEHOST} /usr/bin/svnlook youngest ${REMOTEREPO}`

echo "Local version is ${LOCALVER}"
echo "Remote version is ${REMOTEVER}"

if [ "$REMOTEVER" -gt "$LOCALVER" ];
then
echo "Remote version is greater than local version"
START=$(echo "${LOCALVER} + 1" | /usr/bin/bc -l)
/usr/bin/ssh ${REMOTEUSER}@${REMOTEHOST} /usr/bin/svnadmin dump --incremental --deltas --revision ${START}:${REMOTEVER} ${REMOTEREPO} | /usr/bin/svnadmin load --ignore-uuid ${LOCALREPO}
else
echo "Both local and remote version have the same data"
fi

--------
Schedule with this cron file:
cat myapp_svn_backup.cron
#!/bin/bash
cd /root/backup/[myapp]/svn_logs
logfilename="subversion_backup_"`eval date +%Y%m%d`".log"
/root/backup/[myapp]/run_svn_backup.sh > $logfilename

Local mysql backups (one file per table)

> cat run_mysql_backup.sh
export backupsdir=/root/backup/[myapp]/mysql_backups
export d=`eval date +%Y%m%d`
mkdir -p $backupsdir/$d
for i in `echo "show tables" | mysql -h[host] -u root -p[pass] [dbname]|grep -v Tables_in_`;
do
echo $i; mysqldump --add-drop-table --allow-keywords -q -a -c -h[host] -u root -p[pass] [dbname] $i > $backupsdir/$d/$i.sql
rm -f $backupsdir/$d/$i.sql.gz
gzip $backupsdir/$d/$i.sql
done