Python ETL for Observepoint Folders into PostgreSQL Database

Create the table:

CREATE TABLE public.folders
(
    appsrunning INTEGER,
    appsinqueue INTEGER,
    fname text COLLATE pg_catalog."default",
    apprulefailures INTEGER,
    createdbyusername text COLLATE pg_catalog."default",
    appscount INTEGER,
    accountid INTEGER,
    createdbyuserid INTEGER,
    folderid INTEGER,
    auditsrunning INTEGER,
    createdat TIMESTAMP WITHOUT TIME zone,
    appfailures INTEGER,
    simulationerrors INTEGER,
    simulationsrunning INTEGER,
    domainscount INTEGER
)

Pull the JSON data from the Observepoint API and load:

def loadFolders(c):
    print(':loadFolders() start')

    apiconn = http.client.HTTPSConnection("api.observepoint.com")
    payload = "{}"
    headers = { 'authorization': "api_key yourapikey" }
    endpoint = "/v2/folders"
    apiconn.request("GET", endpoint, payload, headers)
    res = apiconn.getresponse()
    data = res.read()
    de_data = data.decode("utf-8")
    jdata = json.loads(de_data)
   
    for rownum, drow in enumerate(jdata, 1):
        #pprint.pprint(drow)
       
        #create an empty list of full values since calls return partial values
        insertvaluesnames = ['appsRunning','appsInQueue','name','appRuleFailures','createdByUserName','appsCount','accountId','createdByUserId','id','auditsRunning','createdAt','appFailures','simulationErrors','simulationsRunning','domainsCount']
        insertvalues = [None] * len(insertvaluesnames)
        #print('drow:' + str(rownum))
        for dfield in drow:
           
            for insert_list_index,currentvalue in enumerate(insertvaluesnames):
                if dfield == currentvalue:
                    insertvalues[insert_list_index] = drow.get(dfield)

        queryFunc.insertFolder(c,insertvalues)
        #print(insertvalues)
       
    print(':loadFolders() end')

Function used to insert into the PostgreSQL folder table:

def insertFolder(c,a):
    """insertFolder()START  """
    try:
        sql = """
        INSERT INTO folders (appsrunning,appsinqueue,fname,apprulefailures,createdbyusername,
                                appscount,accountid,createdbyuserid,folderid,auditsrunning,
                                createdat,appfailures,simulationerrors,simulationsrunning,domainscount)
             VALUES(%s, %s, %s, %s, %s,
                    %s, %s, %s, %s, %s,
                    %s, %s, %s, %s, %s)
             """

        # create a cursor
        cur = c.cursor()
        cur.execute(sql, a)
        c.commit()
        #print('inserted folder')
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)

Leave a Reply

Your email address will not be published. Required fields are marked *