Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions packages/google-cloud-storage/google/cloud/storage/blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -1271,18 +1271,23 @@ def _handle_filename_and_download(self, filename, *args, **kwargs):
For *args and **kwargs, refer to the documentation for download_to_filename() for more information.
"""

try:
with open(filename, "wb") as file_obj:
with open(filename, "wb") as file_obj:
try:
self._prep_and_do_download(
file_obj,
*args,
**kwargs,
)

except (DataCorruption, NotFound):
# Delete the corrupt or empty downloaded file.
os.remove(filename)
raise
except Exception:
# Remove the (empty or partial) file before re-raising so
# callers never see a corrupt destination file. This extends
# the existing DataCorruption/NotFound cleanup to cover network
# errors, timeouts, and any other unexpected failures.
try:
os.remove(filename)
except OSError:
pass # file may not exist if open() itself failed
raise
Comment thread
RyanCunhaCosta marked this conversation as resolved.

updated = self.updated
if updated is not None:
Expand Down Expand Up @@ -1312,6 +1317,13 @@ def download_to_filename(
If :attr:`user_project` is set on the bucket, bills the API request
to that project.

.. note::
If the download fails for any reason (network error, timeout,
:exc:`~google.cloud.exceptions.NotFound`, etc.), any partially
written destination file is removed before re-raising the
exception, so the filesystem is never left with an empty or
incomplete file.

See a [code sample](https://cloud.google.com/storage/docs/samples/storage-download-encrypted-file#storage_download_encrypted_file-python)
to download a file with a [`customer-supplied encryption key`](https://cloud.google.com/storage/docs/encryption#customer-supplied).

Expand Down
85 changes: 85 additions & 0 deletions packages/google-cloud-storage/tests/unit/test_blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -1962,6 +1962,91 @@ def test_download_to_filename_notfound(self):
stream = blob._prep_and_do_download.mock_calls[0].args[0]
self.assertEqual(stream.name, filename)

def test_download_to_filename_cleans_up_on_connection_error(self):
import requests.exceptions

blob_name = "blob-name"
client = self._make_client()
bucket = _Bucket(client)
blob = self._make_one(blob_name, bucket=bucket)

with mock.patch.object(blob, "_prep_and_do_download"):
blob._prep_and_do_download.side_effect = requests.exceptions.ConnectionError(
"Network is unreachable"
)

filehandle, filename = tempfile.mkstemp()
os.close(filehandle)
self.assertTrue(os.path.exists(filename))

with self.assertRaises(requests.exceptions.ConnectionError):
blob.download_to_filename(filename)

self.assertFalse(os.path.exists(filename))

def test_download_to_filename_cleans_up_on_timeout(self):
import requests.exceptions

blob_name = "blob-name"
client = self._make_client()
bucket = _Bucket(client)
blob = self._make_one(blob_name, bucket=bucket)

with mock.patch.object(blob, "_prep_and_do_download"):
blob._prep_and_do_download.side_effect = requests.exceptions.Timeout(
"Timeout of 120.0s exceeded"
)

filehandle, filename = tempfile.mkstemp()
os.close(filehandle)
self.assertTrue(os.path.exists(filename))

with self.assertRaises(requests.exceptions.Timeout):
blob.download_to_filename(filename)

self.assertFalse(os.path.exists(filename))

def test_download_to_filename_cleans_up_on_generic_exception(self):
blob_name = "blob-name"
client = self._make_client()
bucket = _Bucket(client)
blob = self._make_one(blob_name, bucket=bucket)

with mock.patch.object(blob, "_prep_and_do_download"):
blob._prep_and_do_download.side_effect = RuntimeError("unexpected failure")

filehandle, filename = tempfile.mkstemp()
os.close(filehandle)
self.assertTrue(os.path.exists(filename))

with self.assertRaises(RuntimeError):
blob.download_to_filename(filename)

self.assertFalse(os.path.exists(filename))

def test_download_to_filename_keeps_file_on_success(self):
blob_name = "blob-name"
client = self._make_client()
bucket = _Bucket(client)
blob = self._make_one(blob_name, bucket=bucket)

def _write_data(file_obj, **kwargs):
file_obj.write(b"hello world")

with mock.patch.object(
blob, "_prep_and_do_download", side_effect=_write_data
):
filehandle, filename = tempfile.mkstemp()
os.close(filehandle)
try:
blob.download_to_filename(filename)
self.assertTrue(os.path.exists(filename))
with open(filename, "rb") as f:
self.assertEqual(f.read(), b"hello world")
finally:
if os.path.exists(filename):
os.remove(filename)

def _download_as_bytes_helper(
self, raw_download, timeout=None, single_shot_download=False, **extra_kwargs
):
Expand Down
Loading