Database Reference
In-Depth Information
import auth
# Number of bytes to download per request.
CHUNKSIZE = 1024 * 1024
class GcsReader:
'''Reads files from Google Cloud Storage.
Verifies the presence of files in Google Cloud
Storage. Will download
the files as well if download_dir is not None.
'''
def __init__(self, gcs_bucket, download_dir=None):
self.gcs_service = auth.build_gcs_client()
self.gcs_bucket = gcs_bucket
self.download_dir = download_dir
def make_uri(self, gcs_object):
'''Turn a bucket and object into a Google Cloud
Storage path.'''
return 'gs://%s/%s' % (self.gcs_bucket, gcs_object)
def check_gcs_file(self, gcs_object):
'''Returns a tuple of (GCS URI, size) if the file
is present.'''
try:
metadata = self.gcs_service.objects().get(
bucket=self.gcs_bucket,
object=gcs_object).execute()
uri = self.make_uri(gcs_object)
return (uri, int(metadata.get('size', 0)))
except HttpError, err:
# If the error is anything except a 'Not Found'
print the error.
if err.resp.status <> 404:
print err
return (None, None)
Search WWH ::




Custom Search