Backup of Domo Cards¶

Domo reference_Tool)

Further documentation in Confluence

The purpose of this notebook is to create ".json" backup files for each card created in Domo.

It is possible then for deleted cards to be restored from these backups.

For more details see this Google doc

Setup¶

In [ ]:

!pip install icecream

!pip install icecream

Collecting icecream
  Downloading https://files.pythonhosted.org/packages/31/cc/5454531fe9ae123720b496fdea806e282843d6e75e5718a5e8b1d8e5c47f/icecream-2.1.0-py2.py3-none-any.whl
Collecting executing>=0.3.1
  Downloading https://files.pythonhosted.org/packages/e1/a6/07d28b53b1fab42985cba6b704d685a60a2e3a5efce4cfaaad42a4494bd8/executing-0.6.0-py2.py3-none-any.whl
Requirement already satisfied: pygments>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from icecream) (2.6.1)
Collecting colorama>=0.3.9
  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl
Collecting asttokens>=2.0.1
  Downloading https://files.pythonhosted.org/packages/16/d5/b0ad240c22bba2f4591693b0ca43aae94fbd77fb1e2b107d54fff1462b6f/asttokens-2.0.5-py2.py3-none-any.whl
Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from asttokens>=2.0.1->icecream) (1.15.0)
Installing collected packages: executing, colorama, asttokens, icecream
Successfully installed asttokens-2.0.5 colorama-0.4.4 executing-0.6.0 icecream-2.1.0

In [ ]:

import pandas as pd
import pathlib
import datetime
import os
import subprocess
import sqlite3
from icecream import ic

import pandas as pd import pathlib import datetime import os import subprocess import sqlite3 from icecream import ic

In [ ]:

from google.colab import drive
drive.mount('/content/drive')

from google.colab import drive drive.mount('/content/drive')

Mounted at /content/drive

In [ ]:

# create folder name
# edit this cell if you would like to manually assign a folder name where 
# backups are saved
save_folder_name = str(datetime.datetime.now().date())
#save_folder_name = "2021-06-02"

# create folder name # edit this cell if you would like to manually assign a folder name where # backups are saved save_folder_name = str(datetime.datetime.now().date()) #save_folder_name = "2021-06-02"

In [ ]:

# create folder to save files
working_folder = pathlib.Path("/content/drive/MyDrive/Cartology/Domo/08. Backup Domo Cards")

new_save_folder = working_folder / "Backups" / save_folder_name

os.mkdir(new_save_folder)

# create folder to save files working_folder = pathlib.Path("/content/drive/MyDrive/Cartology/Domo/08. Backup Domo Cards") new_save_folder = working_folder / "Backups" / save_folder_name os.mkdir(new_save_folder)

In [ ]:

# create folder to hold command line tool scripts that will run
script_folder = new_save_folder / "scripts"
os.mkdir(script_folder)

# create folder to hold command line tool scripts that will run script_folder = new_save_folder / "scripts" os.mkdir(script_folder)

In [ ]:

# create connection to sqlite database
database_path = working_folder.joinpath("card_backups_v2.db")
cnx = sqlite3.connect(database_path)

# create connection to sqlite database database_path = working_folder.joinpath("card_backups_v2.db") cnx = sqlite3.connect(database_path)

In [ ]:

# Download the tool
!wget https://app.domo.com/labs/java-sdk/latest/domoUtil.jar

# Download the tool !wget https://app.domo.com/labs/java-sdk/latest/domoUtil.jar

--2021-06-18 05:55:10--  https://app.domo.com/labs/java-sdk/latest/domoUtil.jar
Resolving app.domo.com (app.domo.com)... 13.227.211.107, 13.227.211.120, 13.227.211.72, ...
Connecting to app.domo.com (app.domo.com)|13.227.211.107|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 30332253 (29M) [application/x-www-form-urlencoded]
Saving to: ‘domoUtil.jar’

domoUtil.jar        100%[===================>]  28.93M   155MB/s    in 0.2s    

2021-06-18 05:55:10 (155 MB/s) - ‘domoUtil.jar’ saved [30332253/30332253]

Backup Process¶

Download card list csv file¶

In [ ]:

download_script_path = working_folder.joinpath("download_card_csv.script").as_posix()

subprocess.run(["java", "-jar", "domoUtil.jar", "-script", download_script_path])

download_script_path = working_folder.joinpath("download_card_csv.script").as_posix() subprocess.run(["java", "-jar", "domoUtil.jar", "-script", download_script_path])

Out[ ]:

CompletedProcess(args=['java', '-jar', 'domoUtil.jar', '-script', '/content/drive/MyDrive/Cartology/Domo/08. Backup Domo Cards/download_card_csv.script'], returncode=0)

In [ ]:

# get card ID into a list from CSV file

data = pd.read_csv("/content/all_cards.csv")

card_num_list = list(data["Card ID"])
card_num_list = list(set(card_num_list))

# get card ID into a list from CSV file data = pd.read_csv("/content/all_cards.csv") card_num_list = list(data["Card ID"]) card_num_list = list(set(card_num_list))

In [ ]:

# save card list dataset to database
data["download_date"] = str(datetime.datetime.now())

my_columns = list(data.columns)
my_columns = list(map(lambda x: x.lower(), my_columns))
my_columns = list(map(lambda x: x.replace(" ", "_"), my_columns))
my_columns = list(map(lambda x: x.replace("/", "_"), my_columns))

data.columns = my_columns


data.to_sql("cards", 
            con = cnx, 
            if_exists="replace", 
            index = False)
#data.head()

# save card list dataset to database data["download_date"] = str(datetime.datetime.now()) my_columns = list(data.columns) my_columns = list(map(lambda x: x.lower(), my_columns)) my_columns = list(map(lambda x: x.replace(" ", "_"), my_columns)) my_columns = list(map(lambda x: x.replace("/", "_"), my_columns)) data.columns = my_columns data.to_sql("cards", con = cnx, if_exists="replace", index = False) #data.head()

In [ ]:

# Number of cards
len(list(set(card_num_list)))

# Number of cards len(list(set(card_num_list)))

Out[ ]:

517

Helper functions¶

In [ ]:

def create_script(card_number:str):
    """Helper function to create a Domo CLI script that will backup a card in
    json format when executed.
    """

    temp_file = str(card_number) + ".json"
    card_number_export = new_save_folder / temp_file

    template_string = f"""connect -u username -p password -s citrusad-cartology.domo.com
backup-card -i {card_number} -f '{card_number_export.as_posix()}'
quit
"""
    script_name = str(card_number) + ".script"
    full_script_path = script_folder / script_name

    # write string to disc
    f = open(full_script_path.as_posix(), "w")
    f.write(template_string)
    f.close()

    return [template_string, 
            full_script_path.as_posix(),
            card_number_export.as_posix()]



def run_script(path_to_script):
    """
    """

    subprocess.run(["java", "-jar", "domoUtil.jar", "-script", path_to_script])

    return None


def read_json(path_to_json, path_to_script):
    """Attempts to read a json file.
    If unable to read the file attempts to download the json file a maximum of
    5 times.
    """

    i = 0
    while i < 3:
        try:
            with open(path_to_json) as f:
                content = f.readlines()
            return content[0]
            i = 7
        except IOError:
            i = i + 1
            #ic(i)
            #ic(path_to_json)
            #run_script(path_to_script)
    return None



def main_runner(card_number):
    """Does the following for a given card number:
    1. creates and saves backup script
    2. runs backup script
    3. saves backup json file
    """

    string_and_path = create_script(card_number)

    temp_script_contents = string_and_path[0]
    temp_script_path = string_and_path[1]
    temp_json_path = string_and_path[2]

    run_script(temp_script_path)

    temp_json_contents = read_json(temp_json_path, temp_script_path)

    return [temp_script_contents,
            temp_script_path,
            temp_json_path,
            temp_json_contents]

def create_script(card_number:str): """Helper function to create a Domo CLI script that will backup a card in json format when executed. """ temp_file = str(card_number) + ".json" card_number_export = new_save_folder / temp_file template_string = f"""connect -u username -p password -s citrusad-cartology.domo.com backup-card -i {card_number} -f '{card_number_export.as_posix()}' quit """ script_name = str(card_number) + ".script" full_script_path = script_folder / script_name # write string to disc f = open(full_script_path.as_posix(), "w") f.write(template_string) f.close() return [template_string, full_script_path.as_posix(), card_number_export.as_posix()] def run_script(path_to_script): """ """ subprocess.run(["java", "-jar", "domoUtil.jar", "-script", path_to_script]) return None def read_json(path_to_json, path_to_script): """Attempts to read a json file. If unable to read the file attempts to download the json file a maximum of 5 times. """ i = 0 while i < 3: try: with open(path_to_json) as f: content = f.readlines() return content[0] i = 7 except IOError: i = i + 1 #ic(i) #ic(path_to_json) #run_script(path_to_script) return None def main_runner(card_number): """Does the following for a given card number: 1. creates and saves backup script 2. runs backup script 3. saves backup json file """ string_and_path = create_script(card_number) temp_script_contents = string_and_path[0] temp_script_path = string_and_path[1] temp_json_path = string_and_path[2] run_script(temp_script_path) temp_json_contents = read_json(temp_json_path, temp_script_path) return [temp_script_contents, temp_script_path, temp_json_path, temp_json_contents]

Main function run¶

In [ ]:

details_data = []

for card_number in card_num_list:
    temp = main_runner(card_number)
    details_data.append(temp)

details_data = [] for card_number in card_num_list: temp = main_runner(card_number) details_data.append(temp)

Save collected data to database¶

In [ ]:

to_save = pd.DataFrame(details_data)

to_save.columns = ["script_contents", "script_path", "json_path", "json_contents"]
to_save["backup_date"] = str(datetime.datetime.now())

# ensure all fields are strings
to_save = to_save.applymap(str)

# save data to database
to_save.to_sql("backup", 
            con = cnx, 
            if_exists="append", 
            index = False)


to_save.head()

to_save = pd.DataFrame(details_data) to_save.columns = ["script_contents", "script_path", "json_path", "json_contents"] to_save["backup_date"] = str(datetime.datetime.now()) # ensure all fields are strings to_save = to_save.applymap(str) # save data to database to_save.to_sql("backup", con = cnx, if_exists="append", index = False) to_save.head()

In [ ]:

cnx.close()

cnx.close()

In [ ]: