Python Script to Pull JSON from API and Insert into DB

This script is used to populate the jctions table with the journeyids from the journeys table that we just loaded into the PostgreSQL Database.

def loadActions(c):
print(':readActions() start')

cur = c.cursor()
# execute a statement
sql = ('SELECT journeyid from journeys')
print('getting Journeys from DB: ' + sql)
print('getting Actions from OP')
apiconn = http.client.HTTPSConnection("api.observepoint.com")
payload = "{}"
headers = { 'authorization': "api_key yourapikey" }

cur.execute(sql)

for row in cur:
jid = cur.fetchone()
endpoint = "/v2/web-journeys/%s/actions" % jid
apiconn.request("GET", endpoint, payload, headers)
res = apiconn.getresponse()
data = res.read()
de_data = data.decode("utf-8")
jdata = json.loads(de_data)
#print('jid:' + str(jid))

for rownum, drow in enumerate(jdata, 1):
#create an empty list of full values since calls return partial values
insertvaluesnames = ['url','preventnavigation','label','sequence','js','rules','value','action']
insertvalues = [None] * len(insertvaluesnames)
#print('drow:' + str(rownum))
for dfield in drow:
#print(dfield + ':', end='')
#print(drow.get(dfield))

for insert_list_index,currentvalue in enumerate(insertvaluesnames):
if dfield == currentvalue:
insertvalues[insert_list_index] = drow.get(dfield)

insertvalues.insert(0,jid[0])
queryFunc.insertAction(c,insertvalues)
#print (insertvalues)

#if rownum == 1:
# break

print(':loadActions() end')

This insertAction function to write data to the database:

def insertAction(c,a):
    """insertAction()START  """
    try:
        sql = """
        INSERT INTO actions (journeyid, url, preventnavigation, label, sequence,
                                js, rules, value, action)
             VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s)
             """

        # create a cursor
        cur = c.cursor()
        cur.execute(sql, a)
        c.commit()
        #print('inserted action')
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)

Leave a Reply

Your email address will not be published. Required fields are marked *