Database Reference
In-Depth Information
def make_output_dir(self, output_file):
'''Creates an output directory for the downloaded
results.'''
output_dir = os.path.dirname(output_file)
if os.path.exists(output_dir) and
os.path.isdir(output_dir):
# Nothing to do.
return
os.makedirs(output_dir)
def complete_download(self, media):
while True:
# Download the next chunk, allowing 3 retries.
_, done = media.next_chunk(num_retries=3)
if done: return
def download_file(self, gcs_object):
'''Downloads a GCS object to directory
download_dir.'''
output_file_name = os.path.join(self.download_dir,
gcs_object)
self.make_output_dir(output_file_name)
with open(output_file_name, 'w') as out_file:
request = self.gcs_service.objects().get_media(
bucket=self.gcs_bucket, object=gcs_object)
media = MediaIoBaseDownload(out_file, request,
chunksize=CHUNKSIZE)
print 'Downloading:\n%s to\n%s' % (
self.make_uri(gcs_object), output_file_name)
self.complete_download(media)
def read(self, gcs_object):
'''Read the file and returns the file size or
None if not found.'''
uri, file_size = self.check_gcs_file(gcs_object)
if uri is None:
