summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNathan Kinkade <nath@nkinka.de>2013-11-26 14:20:52 +0000
committerNathan Kinkade <nath@nkinka.de>2013-11-26 14:20:52 +0000
commitddcfb32e729ae70de9e65d1f3e88663a7dda6f0a (patch)
tree3a4375335717766daa49c0814037b98841711609
parent4d8cb65c7d43672f7e8af3e18275220efff32b5d (diff)
When a urllib2 returns error code 500, try incrementally sleeping more and more after each of the 3 attempts.
-rwxr-xr-xmiami_dade_clerk_courts.py7
1 files changed, 6 insertions, 1 deletions
diff --git a/miami_dade_clerk_courts.py b/miami_dade_clerk_courts.py
index 5afd525..709f783 100755
--- a/miami_dade_clerk_courts.py
+++ b/miami_dade_clerk_courts.py
@@ -6,6 +6,7 @@ import urllib
import urllib2
import re
import string
+import time
import logging
from BeautifulSoup import BeautifulSoup
from sqlalchemy import create_engine
@@ -135,11 +136,15 @@ def fetch_page(post_fields):
page = response.read()
return page
except (urllib2.HTTPError, urllib2.URLError) as e:
+ attempt = attempt + 1
if hasattr(e, 'reason'):
logging.error('%s: urllib2 error reason: %s' % (current_case, e.reason))
if hasattr(e, 'code'):
logging.error('%s: urllib2 error code: %s' % (current_case, e.code))
- attempt = attempt + 1
+ if e.code == '500':
+ sleeptime = 2 ** attempt * 10
+ time.sleep(sleeptime)
+ logging.error('%s: Trying again after sleeping for %d seconds' % (current_case, sleeptime))
except:
logging.error('%s: fetch_page() exception %s - %s'
% (current_case, sys.exc_info()[0], sys.exc_info()[1]))