Scrape Foursquare API to get venue information

import urllib
import urllib2
import json
import datetime
import pandas as pd
from pandas.io.json import json_normalize
import math
import time
from math import cos
from pandas import DataFrame

### Helper function for converting meters to lat/long

def distcust(p, d, lat_m, long_m):
    lat = p['lat']
    long = p['long']

    lat1 = lat + lat_m * (d / (11100.0/90*1000) * cos(lat))
    long1 = long + long_m * (d / (11100.0/90*1000))

    return {'lat': lat1, 'long': long1}

client_id = "YOUR_CLIENT_ID"
client_secret = "YOUR_CLIENT_SECRET"
#p = {'lat': 37.7833, 'long': -122.4167}    # central San Francisco, at Van Ness and Market
#p = {'lat': 40.783011, 'long': -73.965368} # central NYC, at Central Park
p = {'lat': 42.963601, 'long': -85.66878} # grand rapids, mi  at division and fulton 
distance = 100
limit = 50
gridSize = 10
df = DataFrame()
requested_keys = ["categories","id","location","name"]
category = "bar"
category_id = "4d4b7105d754a06376d81259"

for x in [x1 / 10.0 for x1 in range(-3*gridSize, 3*gridSize)]:
    for y in [y1 / 10.0 for y1 in range(-3*gridSize, 3*gridSize)]:
        center = distcust(p,distance,x,y)
        url = "https://api.foursquare.com/v2/venues/search?ll=%s,%s&intent=browse&radius=%s&categoryId=%s&client_id=%s&client_secret=%s&v=%s" % (center["lat"], center["long"], distance, category_id, client_id, client_secret, time.strftime("%Y%m%d"))
        try:
            req = urllib2.Request(url)
            response = urllib2.urlopen(req)
            data = json.loads(response.read())
            response.close()
            #print data["response"]['venues']
            data = DataFrame(data["response"]['venues'])[requested_keys]


            df2 = DataFrame()
            venue_ids = []
            frames = []

            #print data["id"]
            for d in data["id"]:                
                requested_keys2 = ["id", "price.currency","rating", "likes.count"]

                url2 = "https://api.foursquare.com/v2/venues/%s?client_id=%s&client_secret=%s&v=%s" % (d, client_id, client_secret, time.strftime("%Y%m%d"))
                req2 = urllib2.Request(url2)
                response2 = urllib2.urlopen(req2)
                data2 = json.loads(response2.read())
                response.close()
                ddata = data2['response']               

                nom_data = json_normalize(ddata['venue'])

                if "price.currency" not in nom_data.columns:
                    nom_data["price.currency"] = 'NONE'

                if "rating" not in nom_data.columns:
                    nom_data["rating"] = 'NONE'                 

                venue_ids.append(d)
                frames.append(nom_data[requested_keys2])
                #print "getting attr for %s" % nom_data["name"]
                time.sleep(1)


            df2 = pd.concat(frames, keys=venue_ids)

            mdata = pd.merge(data, df2,how='left',on='id', suffixes=('_x', '_y'))

            #print mdata

            df = df.append(mdata,ignore_index=True)
            #print df

            #df.to_csv("test.csv")

            print center
            time.sleep(1) # stay within API limits
        except Exception, e:
            print e

df = df.drop_duplicates(cols='id',take_last=True)
print df

df["categories"] = df["categories"].apply(lambda x: dict(x[0])['name'])
df["lat"] = df["location"].apply(lambda x: dict(x)["lat"])
df["long"] = df["location"].apply(lambda x: dict(x)["lng"])
df["distance"] = df["location"].apply(lambda x: dict(x)["distance"])
df["checkins"] = df["stats"].apply(lambda x: dict(x)["checkinsCount"])

ordered_df = df[["id_x","name_x","categories","checkins", "distance","lat","long", "price.currency", "rating", "likes.count"]]
ordered_df.to_csv("foursquare_%s_grand_rapids.csv" % category,encoding='utf-8', index=False)