diff options
Diffstat (limited to 'fetch.py')
| -rw-r--r-- | fetch.py | 46 |
1 files changed, 25 insertions, 21 deletions
| @@ -21,25 +21,29 @@ from urllib.request import urlopen | |||
| 21 | 21 | ||
| 22 | 22 | ||
| 23 | def fetch_file(url, verbose=False): | 23 | def fetch_file(url, verbose=False): |
| 24 | """Fetch a file from the specified source using the appropriate protocol. | 24 | """Fetch a file from the specified source using the appropriate protocol. |
| 25 | 25 | ||
| 26 | Returns: | 26 | Returns: |
| 27 | The contents of the file as bytes. | 27 | The contents of the file as bytes. |
| 28 | """ | 28 | """ |
| 29 | scheme = urlparse(url).scheme | 29 | scheme = urlparse(url).scheme |
| 30 | if scheme == 'gs': | 30 | if scheme == "gs": |
| 31 | cmd = ['gsutil', 'cat', url] | 31 | cmd = ["gsutil", "cat", url] |
| 32 | try: | 32 | try: |
| 33 | result = subprocess.run( | 33 | result = subprocess.run( |
| 34 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, | 34 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True |
| 35 | check=True) | 35 | ) |
| 36 | if result.stderr and verbose: | 36 | if result.stderr and verbose: |
| 37 | print('warning: non-fatal error running "gsutil": %s' % result.stderr, | 37 | print( |
| 38 | file=sys.stderr) | 38 | 'warning: non-fatal error running "gsutil": %s' |
| 39 | return result.stdout | 39 | % result.stderr, |
| 40 | except subprocess.CalledProcessError as e: | 40 | file=sys.stderr, |
| 41 | print('fatal: error running "gsutil": %s' % e.stderr, | 41 | ) |
| 42 | file=sys.stderr) | 42 | return result.stdout |
| 43 | sys.exit(1) | 43 | except subprocess.CalledProcessError as e: |
| 44 | with urlopen(url) as f: | 44 | print( |
| 45 | return f.read() | 45 | 'fatal: error running "gsutil": %s' % e.stderr, file=sys.stderr |
| 46 | ) | ||
| 47 | sys.exit(1) | ||
| 48 | with urlopen(url) as f: | ||
| 49 | return f.read() | ||