问将数据传输文件v2加载到大查询中

EN

import csv,json
import glob
data = []

for i in glob.glob('*.csv'):
  print i
  b = i.split("_")
  print b[2]
  with open(i, 'rb') as f:
    reader = csv.reader(f)
    row1 = next(reader)
  title = [w.replace(' ', '_').replace('/', '_').replace(':', '_').replace('(', '_').replace(')', '').replace("-", "_") for w in row1]
  print title

  for a in title:
    j={"name":"{0}".format(a),"type":"string","mode":"nullable"}
    print j
    if j not in data:
      data.append(j)

  with open('schema_' + b[2] + '.json', 'w') as outfile:
    json.dump(data, outfile)

#!/bin/bash

prep_files() {
date=$(echo "$f" | cut -d'_' -f4 | cut -c1-8)
echo "$n"
table_name=$(echo "$f" | cut -d'_' -f1-3)
bq --nosync load --field_delimiter=',' DCM_V2."$table_name""_""$date" "$var" ./schema/v2/schema_"$n".json
}


num=1

for var in $(gsutil ls gs://import-log/01_v2/*.csv.gz)
    do
       if test $num -lt 10000
       then
          echo "$var"
          f=$(echo "$var" | cut -d'/' -f5)
          n=$(echo "$f" |  cut -d'_' -f3)
          echo "$n"
          prep_files
          num=$(($num+1))
       else
          echo -e "Wait the next day"
          echo "$num"
          sleep $(( $(date -d 'tomorrow 0100' +%s) - $(date +%s) ))
          num=0
        fi
    done


 echo 'Import done'

id,url
1,http://www.google.com
2,"http://www.google.com/test1,test2,test3"