#SCRAPE BOM

#requires RCurl
#AusInfo:  collation of Australian stations ids from 	 
   #paste("http://www.bom.gov.au/cgi-bin/climate/hqsites/site_networks.cgi?variable=meanT&period=monthly&state=",aus.state[i],sep="")


library(RCurl)

# FUNCTION #haven't checked whether data is always continuous
	get.bom=function(id) { #retrieves station data 
	 # id=AusInfo$id[40];id 
	  i=grep(id,AusInfo$id); i
 	   loc=paste("http://www.bom.gov.au/cgi-bin/climate/hqsites/site_networks.cgi?variable=meanT&period=monthly&state=",
		AusInfo$state[i],sep="")
	   test=getURL(loc, httpheader = c("User-Agent" = "R (2.9.0)") )
	   Sys.sleep(1)   
	    loc=paste("http://www.bom.gov.au/cgi-bin/climate/hqsites/site_data.cgi?variable=meanT&area=",AusInfo$state[i],"&station=",
	        AusInfo$id[i],"&period=monthly",sep="");loc
            test=getURL(loc,httpheader = c("User-Agent" = "R (2.9.0)") )
	    Sys.sleep(1)  
	  loc=paste( "http://www.bom.gov.au/climate/change/hqsites/data/temp/meanT.",AusInfo$id[i],".monthly.txt",sep="")
	   station=read.table(loc,skip=1)
	   temp= station[,3] > 99999
	   station[temp,3]=NA
	   x= ts(station[,3],start=c(as.numeric(substr(station[1,1],1,4)), as.numeric(substr(station[1,1],5,6) ) ),freq=12)  
	   return(x)
	}


	AusInfo=read.csv("http://www.climateaudit.org/data/station/australia/AusInfo.csv",sep="\t",colClasses="character")

	
	month=c(paste("0",1:9,sep=""),10:12)

	get.aus= function() {
	  dummy=NULL
	  for(i in 1:12) {
	   loc=paste("http://www.bom.gov.au/web01/ncc/www/cli_chg/timeseries/tmean/",month[i],"/aus/latest.txt",sep="")
	   fred=read.table(loc)
	   dummy=ts.union(dummy,ts(fred[,2],start=1950))
	   }
	  x=ts( c(t(dummy)),start=1950,freq=12)
 	  return(x)
	}

##EXAMPLES
 # australia=get.aus()

 # source("http://www.climateaudit.org/scripts/station/australia/functions.bom.txt")
 # station= get.bom(id=AusInfo$id[i])

 #Collation  
  #K=nrow(AusInfo);K #103
  #Aus=rep(list(NA),K)
  #for (i in 1:K) 	Aus[[i]]=get.bom(id=AusInfo$id[i])
  #names(Aus)=AusInfo$id
  #  save(Aus,file="d:/climate/data/station/australia/Aus.tab")