", "", $1) == 1 && entry == 1{# if (D == 1){print "title block: "$0;} gsub("

#
# BSD license : http://creativecommons.org/licenses/BSD/
#
# Copyright (c) 2009, Judith Elaine Bush
# All rights reserved.
# Redistribution and use in source and binary forms, with or without modification, are permitted  provided that the following conditions are met:
#Ą	Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 
#Ą	Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 
#Ą	Neither the name of Judith Elaine Bush nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


BEGIN {#
	D = 0;
	print "" > "Tweets.txt";
	print "" > "CleanTweets.txt";
	print "Twitter handle\treplied to" > "Handle_RepliedTo.txt";
	print "Twitter handle\tcited" > "Handle_Cited.txt"; 
	print "Twitter handle\tmentioned" > "Handle_Mentioned.txt"; 
	print "Twitter handle\ttime (epoch seconds)" > "Handle_Epoch.txt"; 

	Mentions["startup"]="initialization"; delete Mentions; # Mentions is now an array
	entry=0;
}

# I'm using the title instead of the message because the message has Twitter identities 
# marked up in entity encoded HTML. Unfortunately there's also a title for the document:
# we want the title child of entry. Yeah, i should do this in an XML aware environment.

gsub("<entry>", "", $1) == 1{entry=1};
gsub("</entry>", "", $1) == 1{entry=0};

gsub ("<published>", "", $0) ==1 && entry == 1{# 
	gsub("</published>","",$0);
	ZuluDate = $0; # eg: 2009-06-10T21:36:45Z
#	print ZuluDate",1" >> "Zuludate.csv";
	split($1,Zulu,"T"); #eg Zulu[1] = 2009-06-10 & Zulu[2]=21:36:45Z
	split(Zulu[1],Date,"-");
	gsub("Z","",Zulu[2]);
	split(Zulu[2],Time,":");
	#Turns datespec into a time stamp of the same form as returned  by  systime().
	#The  datespec  is  a string  of  the form YYYY MM DD HH MM SS[ DST]. 
	TS=mktime(Date[1]" "Date[2]" "Date[3]" "Time[1]" "Time[2]" "Time[3]);
	#  %s    is replaced by the number of seconds since the Epoch, UTC (see mktime(3)).
	epoch = strftime("%s",TS );

}	
	

gsub("<title>", "", $1) == 1 && entry == 1{#
	if (D == 1){print "title block: "$0;}
	gsub("</title>", "", $0); 
	tweet=$0;
	
	# REPLIES: we'll assume replies begin with a handle
	reply = index($1,"@"); # if reply == 1, this is a reply
	if (reply == 1){ReplyTo = $1}; 

    # ReTweets: we'll assume retweets (disseminater) begin with RT & will have a cited handle next
    if ($1 == "RT" || $1 == "rt"){#
    	retweet = 1;
		if (index($2,"@") == 1){#
			#last character may be punctuation
#			gsub(/[[:punct:]]$/,"",$2); ## Thanks to @_cb_ i won't make this mistake
			gsub(/[~`!$%^&*()-+={\[}\]|\\:;"'<,>\.\?/]$/,"",$2);
			Cited = $2;
		} 
		else {Cited = "missing"}; 
    } 
    else {retweet = 0};

	# Mentions & clean up: 
	# capture the entirety of the network 
	# and pull out handles & tags for wordles, etc
	i=1
	while (i <= NF){# move through all the fields
		if (index($i,"@") == 1){#we probably have a handle
			#last character may be punctuation
#			gsub(/[[:punct:]]$/,"",$i); # Thanks to @_cb_ i won't make this mistake
			gsub(/[~`!$%^&*()-+={\[}\]|\\:;"'<,>\.\?/]$/,"",$i);

			Mentions[i]=$i; #this creates an array of all handles in the tweet
			$i=""; # purging handles
		}
		if (index($i,"#") == 1){#we probably have a tag
			Tags[i]=$i; #this creates an array of all tags in the tweet
			$i=""; # purging tags
		}
		i++;
	};  
	clean_tweet=$0;

	if (D == 1){print "Title ends: "$0;}

}


# Parsing the Atom author block which comes after the title block in the entry
$1 == "<author>" {# This is unnecessary at this point.
	next};
gsub("<name>", "", $1) == 1{# name has the form handle (display name)
	gsub("</name>", "", $0);
	handle=$1;
	name=$0;
}
gsub("<uri>", "", $1) == 1{#
	gsub("</uri>", "", $0); 
	uri=$0;
}
	
	
# When we reach the end of author block, we have both the raw tweet and the author 
# information. Now to analyze. Note
# Network graph: http://manyeyes.alphaworks.ibm.com/manyeyes/page/Network_Diagram.html
# Block Histogram: http://manyeyes.alphaworks.ibm.com/manyeyes/page/Block_Histogram.html
	
$1 == "</author>" {# At the end of the block, create the key & count things
	if (D == 1){print "/Author block";}

	key="<a href=\""uri"\">"name"</a>";
	handle_array[handle]++; # count of tweets by handle
	author_array[key]++  ; # count of tweets by HTML of author

    if (reply == 1){#
    	handle_replies[handle]++;# count of reply messages by handle
    	print "@"handle"\t"ReplyTo >> "Handle_RepliedTo.txt"; # for network graph
    }
	if (retweet == 1){#
		handle_retweets[handle]++;# count of retweets by handle
		print "@"handle"\t"Cited >> "Handle_Cited.txt"; # for network graph		
	}
	if (asort(Mentions) != 0){# we have a populated array
		handle_mentions[handle]++;# count of tweets with mentions by handle
		for (n in Mentions){# loop through all mentions
			print "@"handle"\t"Mentions[n] >> "Handle_Mentioned.txt"; # for network graph		
		}
	}
	
	
	delete Mentions;# deletes all values from the Mentions array
	delete Tags; # deletes all values from the Tags array
	
	# this file of tweet texts can be used for wordles or similar
	print "@"handle"\t"epoch >> "Handle_Epoch.txt"; #for (frequency) block histogram
	print tweet >> "Tweets.txt" ;
	print clean_tweet >> "CleanTweets.txt";
	
	if (D == 1){print "/Author ends";}

}

	
END {# Print to destination files
	print "Twitter handle\ttweet count\treplies\tretweets\tmentions" > "HandleTweetCount.txt";
	for (h in handle_array) {# The handle array has all the handles of tweet authors
		# We want zeros and not nulls in the array for handles without certain activity
		handle_replies[h]=handle_replies[h]+0;
		handle_retweets[h]=handle_retweets[h]+0;
		handle_mentions[h]=handle_mentions[h]+0; 
		print "@"h"\t"handle_array[h]"\t"handle_replies[h]"\t"handle_retweets[h]"\t"handle_mentions[h] >> "HandleTweetCount.txt";

	}
	for (name in author_array) {print author_array[name]" "name > "AuthorTweetCount.html"};


	}