'''
Combining all files while extracting data & removing duplicates
'''
import time

days = ['0629', '0630', '0701', '0702', '0703', '0704', '0705']
times = ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', \
         '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', \
         '22', '23']
fOut = 'sensorsAll.txt'
data = {}

for i in days:
    for j in times:
        fIn = '2012' + i + '_' + j + '00_pre.txt'
        print fIn
        fdIn = open(fIn, 'r')
        for line in fdIn:
            tmpLine = line.split(',') #Comma seperated
            strTime = tmpLine[1] + ' ' + tmpLine[2]
            structTime = time.strptime(strTime, '%m/%d/%Y %H:%M')
            sampleTime = time.mktime(structTime) 
            station = tmpLine[0]
            if sampleTime in data:
                if station in data[sampleTime]: continue #Duplicate
                data[sampleTime][station] = str(sampleTime) + ',' + line
            else: 
                data[sampleTime] = {station: (str(sampleTime) + ',' + line)}
        fdIn.close()

times = data.keys()
times.sort()
fdOut = open(fOut, 'w')
stationNames = {} #[interval, lastSample]
for i in times:
    for j in data[i]:
        if j not in stationNames: 
            stationNames[j] = [-1, i]
        else: stationNames[j] = [i - stationNames[j][1], i]
        fdOut.write(data[i][j])
fdOut.close()

print 'No of stations', len(stationNames)
mini = 9999999999
maxi = -1
for i in stationNames:
    if mini > stationNames[i][0]: mini = stationNames[i][0]
    if maxi < stationNames[i][0]: maxi = stationNames[i][0]
print 'Min sample interval', mini
print 'Max sample interval', maxi
