Database Reference
In-Depth Information
print "[%d] STARTING on %s" % (self.partition_id,
self.gcs_reader.make_uri(self.gcs_object_glob))
job_done = False
shard_index = 0
while True:
file_size = self.read_shard(shard_index)
if file_size is not None:
# Found a new file, save it, and start looking
for the next one.
shard_index += 1
elif job_done: break
else:
# Check whether the job is done. If the job is
done, we don't
# want to exit immediately; we want to check
one more time
# for files.
job_done = self.job_runner.get_job_state() ==
'DONE'
if not job_done:
# Didn't find a new path, and the job is
still running,
# so wait a few seconds and try again.
time.sleep(5)
print "[%d] DONE. Read %d files" %
(self.partition_id, shard_index)
def make_extract_config(source_project_id,
source_dataset_id,
source_table_id,
destination_uris):
'''Creates a dict containing an export job
configuration.'''
source_table_ref = {
'projectId': source_project_id,
'datasetId': source_dataset_id,
