Browse Source

Updatesnapshot close #50 (#51)

* add folder path functionality

* update help

* clarify default and  add example
master
Gerry Nelson 6 years ago
committed by GitHub
parent
commit
826fd19699
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 194
      EXAMPLES.md
  2. BIN
      __pycache__/sharedfunctions.cpython-36.pyc
  3. 102
      snapshotreports.py

194
EXAMPLES.md

@ -10,100 +10,100 @@ The following examples are Linux.
**loginviauathinfo.py** **loginviauathinfo.py**
```bash ```bash
# Login to Viya using default authinfo file ~/.authinfo # Login to Viya using default authinfo file ~/.authinfo
./loginviauthinfo.py ./loginviauthinfo.py
# Login to Viya using an authinfo file # Login to Viya using an authinfo file
./loginviauthinfo.py -f ~/.authinfo_geladm ./loginviauthinfo.py -f ~/.authinfo_geladm
``` ```
**callrestapi.py** **callrestapi.py**
```bash ```bash
#return all the rest calls that can be made to the folders endpoint #return all the rest calls that can be made to the folders endpoint
./callrestapi.py -e /folders -m get ./callrestapi.py -e /folders -m get
#return the json for all the folders/folders #return the json for all the folders/folders
./callrestapi.py -e /folders/folders -m get ./callrestapi.py -e /folders/folders -m get
#return simple text for all the folders/folders #return simple text for all the folders/folders
./callrestapi.py -e /folders/folders -m get -o simple ./callrestapi.py -e /folders/folders -m get -o simple
#rest calls often limit the results returned the text output will tell you returned and total available items #rest calls often limit the results returned the text output will tell you returned and total available items
#in this call set a limit above the total items to see everything #in this call set a limit above the total items to see everything
./callrestapi.py -e /folders/folders?limit=500 -m get -o simple ./callrestapi.py -e /folders/folders?limit=500 -m get -o simple
#return the json for all the identities #return the json for all the identities
./callrestapi.py -e /identities/identities -m get ./callrestapi.py -e /identities/identities -m get
#return the json for all the identities output to a file #return the json for all the identities output to a file
./callrestapi.py -e /identities/identities -m get > identities.json ./callrestapi.py -e /identities/identities -m get > identities.json
#refresh the identities cache #refresh the identities cache
./callrestapi.py -e /identities/userCount -m get ./callrestapi.py -e /identities/userCount -m get
./callrestapi.py -e /identities/cache/refreshes -m post ./callrestapi.py -e /identities/cache/refreshes -m post
#return basic content using accept response type #return basic content using accept response type
./callrestapi.py -m get -e /identities/users/sasadm -a "application/vnd.sas.identity.basic+json" ./callrestapi.py -m get -e /identities/users/sasadm -a "application/vnd.sas.identity.basic+json"
``` ```
**getfolderid.py** **getfolderid.py**
```bash ```bash
# pass the folder path and return the folder id and uri # pass the folder path and return the folder id and uri
./getfolderid.py -f /gelcontent ./getfolderid.py -f /gelcontent
``` ```
**getruleid.py** **getruleid.py**
```bash ```bash
getruleid.py -u /SASVisualAnalytics/** -p "authenticatedUsers" getruleid.py -u /SASVisualAnalytics/** -p "authenticatedUsers"
``` ```
**deletefolder.py** **deletefolder.py**
```bash ```bash
# delete a folder based on its path # delete a folder based on its path
./deletefolder.py -f /gelcontent ./deletefolder.py -f /gelcontent
``` ```
**getconfigurationproperties.py** **getconfigurationproperties.py**
```bash ```bash
#return a set of configuration properties #return a set of configuration properties
./getconfigurationproperties.py -c sas.identities.providers.ldap.user -o simple ./getconfigurationproperties.py -c sas.identities.providers.ldap.user -o simple
``` ```
**createdomain.py and updatedomain.py** **createdomain.py and updatedomain.py**
```bash ```bash
#create a domain using createdomain #create a domain using createdomain
./createdomain.py -t password -d test -u sasadm -p lnxsas -g "SASAdministrators,HR ,Sales" ./createdomain.py -t password -d test -u sasadm -p lnxsas -g "SASAdministrators,HR ,Sales"
#Update an existing domain to add credentials from a csv file #Update an existing domain to add credentials from a csv file
./updatedomain.py -d LASRAuth -f /tmp/myusers.csv ./updatedomain.py -d LASRAuth -f /tmp/myusers.csv
csv file format csv file format
no header row no header row
column1 is userid column1 is userid
column2 is password column2 is password
column3 is identity column3 is identity
column4 is identity type (user or group) column4 is identity type (user or group)
For example: For example:
myuserid,mypass,Sales,group myuserid,mypass,Sales,group
acct1,pw1,sasadm,user acct1,pw1,sasadm,user
#create a domain using callrestapi. Last part of endpoint is domain name #create a domain using callrestapi. Last part of endpoint is domain name
./callrestapi.py -e /credentials/domains/<newdomain> -m post -i domain.json ./callrestapi.py -e /credentials/domains/<newdomain> -m post -i domain.json
INPUT JSON must be formatted as the endpoint expects. Example: INPUT JSON must be formatted as the endpoint expects. Example:
{ {
"id": "<newdomain>", "id": "<newdomain>",
"type": "password" "type": "password"
} }
``` ```
**createpublishdest.py** **createpublishdest.py**
@ -111,7 +111,7 @@ INPUT JSON must be formatted as the endpoint expects. Example:
```bash ```bash
# List all existing publishing destinations # List all existing publishing destinations
./callrestapi.py -m get -e /modelPublish/destinations -o simple ./callrestapi.py -m get -e /modelPublish/destinations -o simple
# Create new publishing destinations - one example each for CAS, Hadoop and Teradata # Create new publishing destinations - one example each for CAS, Hadoop and Teradata
./createpublishdest.py cas -n newcasdest -s cas-shared-default -c mycaslib -t thetable ./createpublishdest.py cas -n newcasdest -s cas-shared-default -c mycaslib -t thetable
./createpublishdest.py hadoop -n newhadoopdest -s cas-shared-default -c myhadoop -hd /mydir ./createpublishdest.py hadoop -n newhadoopdest -s cas-shared-default -c myhadoop -hd /mydir
@ -128,114 +128,114 @@ INPUT JSON must be formatted as the endpoint expects. Example:
**testfolderaccess.py** **testfolderaccess.py**
```bash ```bash
#test folder access #test folder access
./root/admin/pyviyatools/testfolderaccess.py -f '/gelcontent' -p Sales -m read -s grant -q ./root/admin/pyviyatools/testfolderaccess.py -f '/gelcontent' -p Sales -m read -s grant -q
``` ```
**listrules.py** **listrules.py**
```bash ```bash
# Display all sasadministrator rules # Display all sasadministrator rules
./listrules.py --p SASadministrators -o simple ./listrules.py --p SASadministrators -o simple
# Display all rules that contain SASVisual in the URI # Display all rules that contain SASVisual in the URI
./listrules.py -u SASVisual -o simple ./listrules.py -u SASVisual -o simple
``` ```
**createfolders.py** **createfolders.py**
```bash ```bash
# Create folders from a CSV file # Create folders from a CSV file
./createfolders.py" -f /tmp/newfolders.csv ./createfolders.py" -f /tmp/newfolders.csv
FORMAT OF CSV file folder path (parents must exist), description FORMAT OF CSV file folder path (parents must exist), description
/RnD, Folder under root for R&D /RnD, Folder under root for R&D
/RnD/reports, reports /RnD/reports, reports
/RnD/analysis, analysis /RnD/analysis, analysis
/RnD/data plans, data plans /RnD/data plans, data plans
/temp,My temporary folder /temp,My temporary folder
/temp/mystuff, sub-folder /temp/mystuff, sub-folder
``` ```
**updatepreferences.py** **updatepreferences.py**
```bash ```bash
# Update the theme for the user sasadm # Update the theme for the user sasadm
./updatepreferences.py -t user -tn sasadm -pi OpenUI.Theme.Default -pv sas_hcb ./updatepreferences.py -t user -tn sasadm -pi OpenUI.Theme.Default -pv sas_hcb
``` ```
**explainaccess.py** **explainaccess.py**
```bash ```bash
#Explain direct and indirect permissions on the folder /folderA/folderB, no header row. For folders, conveyed permissions are shown by default. #Explain direct and indirect permissions on the folder /folderA/folderB, no header row. For folders, conveyed permissions are shown by default.
./explainaccess.py -f /folderA/folderB ./explainaccess.py -f /folderA/folderB
#As above but for a specific user named Heather #As above but for a specific user named Heather
./explainaccess.py -f /folderA/folderB -n Heather -t user ./explainaccess.py -f /folderA/folderB -n Heather -t user
#As above with a header row #As above with a header row
./explainaccess.py -f /folderA/folderB --header ./explainaccess.py -f /folderA/folderB --header
#As above with a header row and the folder path, which is useful if you concatenate sets of results in one file #As above with a header row and the folder path, which is useful if you concatenate sets of results in one file
./explainaccess.py -f /folderA/folderB -p --header ./explainaccess.py -f /folderA/folderB -p --header
#As above showing only rows which include a direct grant or prohibit #As above showing only rows which include a direct grant or prohibit
./explainaccess.py -f /folderA/folderB --direct_only ./explainaccess.py -f /folderA/folderB --direct_only
#Explain direct and indirect permissions on a service endpoint. Note in the results that there are no conveyed permissions. By default they are not shown for URIs. #Explain direct and indirect permissions on a service endpoint. Note in the results that there are no conveyed permissions. By default they are not shown for URIs.
./explainaccess.py -u /SASEnvironmentManager/dashboard ./explainaccess.py -u /SASEnvironmentManager/dashboard
#As above but including a header row and the create permission, which is relevant for services but not for folders and other objects #As above but including a header row and the create permission, which is relevant for services but not for folders and other objects
./explainaccess.py -u /SASEnvironmentManager/dashboard --header -l read update delete secure add remove create ./explainaccess.py -u /SASEnvironmentManager/dashboard --header -l read update delete secure add remove create
#Explain direct and indirect permissions on a report, reducing the permissions reported to just read, update, delete and secure, since none of add, remove or create are applicable to a report. #Explain direct and indirect permissions on a report, reducing the permissions reported to just read, update, delete and secure, since none of add, remove or create are applicable to a report.
./explainaccess.py -u /reports/reports/id --header -l read update delete secure ./explainaccess.py -u /reports/reports/id --header -l read update delete secure
#Explain direct and indirect permissions on a folder expressed as a URI. Keep the default permissions list, but for completeness we must also specify -c true to request conveyed permissions be displayed, as they are not displayed by default for URIs. #Explain direct and indirect permissions on a folder expressed as a URI. Keep the default permissions list, but for completeness we must also specify -c true to request conveyed permissions be displayed, as they are not displayed by default for URIs.
./explainaccess.py -u /folders/folders/id --header -p -c true ./explainaccess.py -u /folders/folders/id --header -p -c true
``` ```
**getpath.py** **getpath.py**
```bash ```bash
# Get folder path for an object (can be a folder, report or any other object which has a folder path) # Get folder path for an object (can be a folder, report or any other object which has a folder path)
./getpath.py -u /folders/folders/id ./getpath.py -u /folders/folders/id
./getpath.py -u /reports/reports/id ./getpath.py -u /reports/reports/id
``` ```
**listmemberswithpath.py** **listmemberswithpath.py**
```bash ```bash
# Return list of members of a folder identified by objectURI # Return list of members of a folder identified by objectURI
./listmemberswithpath.py -u /folders/folders/id ./listmemberswithpath.py -u /folders/folders/id
# Return list of all members of a folder identified by objectURI, recursively searching subfolders # Return list of all members of a folder identified by objectURI, recursively searching subfolders
./listmemberswithpath.py -u /folders/folders/id -r ./listmemberswithpath.py -u /folders/folders/id -r
``` ```
**listcaslibs.py and listtables.py** **listcaslibs.py and listtables.py**
```bash ```bash
# Return list of all CAS libraries on all servers # Return list of all CAS libraries on all servers
./listcaslibs.py ./listcaslibs.py
# Return list of all CAS tables in all CAS libraries on all servers # Return list of all CAS tables in all CAS libraries on all servers
./listcastables.py ./listcastables.py
``` ```
**listcaslibsandeffectiveaccess.py** **listcaslibsandeffectiveaccess.py**
```bash ```bash
# Return list of all effective access on all CAS libraries on all servers # Return list of all effective access on all CAS libraries on all servers
./listcaslibsandeffectiveaccess.py ./listcaslibsandeffectiveaccess.py
# Return list of all effective access on all CAS tables in all CAS libraries on all servers # Return list of all effective access on all CAS tables in all CAS libraries on all servers
./listcastablesandeffectiveaccess.py ./listcastablesandeffectiveaccess.py
``` ```
**listgroupsandmembers.py** **listgroupsandmembers.py**
```bash ```bash
# Return list of all groups and all their members # Return list of all groups and all their members
./listgroupsandmembers.py ./listgroupsandmembers.py
``` ```
**importpackages.py** **importpackages.py**
```bash ```bash
@ -255,22 +255,26 @@ importpackages.py -d /tmp/mypackage -q
```bash ```bash
# list log files, created by the /jobexecution service older than 6 days old. # list log files, created by the /jobexecution service older than 6 days old.
./listfiles.py -n log -p /jobExecution -d 6 -o csv ./listfiles.py -n log -p /jobExecution -d 6 -o csv
``` ```
**archivefiles.py** **archivefiles.py**
```bash ```bash
# read log files from job execution and add archive them to an os directory # read log files from job execution and add archive them to an os directory
./archivefiles.py -n log -d 6 -p /job -fp /tmp ./archivefiles.py -n log -d 6 -p /job -fp /tmp
``` ```
**snapshotreports.py** **snapshotreports.py**
```bash ```bash
#his tool will export all the reports in your viya system to there own #his tool will export all the reports in your viya system to there own
# individual json file in a directory # individual json file in a directory
# optionally you can pass in folder path and only export reports that
# are located under that folder
./snapshotreports.py -c 10 -d ~/snapshot
./snapshotreports.py -c 10 -d ~/snapshot/salesreports -f /gelcontent/sales
./snapshotreports.py -c 10 -d ~/snapshot
``` ```
**creategroups.py** **creategroups.py**
@ -288,8 +292,8 @@ importpackages.py -d /tmp/mypackage -q
./creategroups.py -f /tmp/newgroups.csv ./creategroups.py -f /tmp/newgroups.csv
Format of csv file is four columns Format of csv file is four columns
Column 1 group id Column 1 group id
Column 2 group name Column 2 group name
Column 3 is a description Column 3 is a description
Column 4 member id Column 4 member id
@ -297,7 +301,7 @@ For example:
group2,"Group 2","My Group 2" group2,"Group 2","My Group 2"
group3,"Group 3","My Group3",geladm group3,"Group 3","My Group3",geladm
group1,"Group 1","group 1" group1,"Group 1","group 1"
``` ```
**applyfolderauthorization.py** **applyfolderauthorization.py**
@ -311,15 +315,15 @@ group1,"Group 1","group 1"
Format of input csv file is 6 columns Format of input csv file is 6 columns
Column 1 is the full path to the folder Column 1 is the full path to the folder
Column 2 is the principal type Column 2 is the principal type
Column 3 is the principal name Column 3 is the principal name
Column 4 is the access setting (grant or prohibit) Column 4 is the access setting (grant or prohibit)
Column 5 is the permissions on the folder Column 5 is the permissions on the folder
Column 6 is the conveyed permissions on the folder's contents Column 6 is the conveyed permissions on the folder's contents
For example: For example:
/gelcontent/GELCorp/Marketing/Reports,group,Marketing,grant,"read,add,remove","read,update,delete,add,remove" /gelcontent/GELCorp/Marketing/Reports,group,Marketing,grant,"read,add,remove","read,update,delete,add,remove"
/gelcontent/GELCorp/Marketing/Reports,user,Douglas,grant,"read,update,add,remove,delete,secure","read,update,add,remove,delete,secure" /gelcontent/GELCorp/Marketing/Reports,user,Douglas,grant,"read,update,add,remove,delete,secure","read,update,add,remove,delete,secure"
/gelcontent/GELCorp/Marketing/Analyses,group,Marketing,grant,"read,add,remove","read,update,delete,add,remove" /gelcontent/GELCorp/Marketing/Analyses,group,Marketing,grant,"read,add,remove","read,update,delete,add,remove"
/gelcontent/GELCorp/Marketing/Work in Progress,group,Marketing,grant,"read,update,add,remove,delete,secure","read,update,add,remove,delete,secure" /gelcontent/GELCorp/Marketing/Work in Progress,group,Marketing,grant,"read,update,add,remove,delete,secure","read,update,add,remove,delete,secure"
``` ```

BIN
__pycache__/sharedfunctions.cpython-36.pyc

Binary file not shown.

102
snapshotreports.py

@ -8,16 +8,18 @@
# individual json file in a directory. # individual json file in a directory.
# #
# The purpose of the tools is to be able to have a granular backup of reports # The purpose of the tools is to be able to have a granular backup of reports
# so that you could restore an individual report to a system. Something that # so that you could restore an individual report to a system. Something that
# is not currently supported by Viya backup or promotion # is not currently supported by Viya backup or promotion
# #
# example # example
# #
# save each to their own package all reports that have changed in the last 10 days # save each to their own package all reports that have changed in the last 10 days
# snapshotreports.py -c 10 -d ~/snapshot # snapshotreports.py -c 10 -d ~/snapshot
# #
# Change History # Change History
# #
# 16may2020 add folder path to report name
# 16may2020 allow to subset reports exported by the path of the report folder
# #
# Copyright © 2019, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. # Copyright © 2019, SAS Institute Inc., Cary, NC, USA. All Rights Reserved.
# #
@ -37,21 +39,22 @@
# Import Python modules # Import Python modules
import argparse, sys, subprocess, uuid, time, os, glob import argparse, sys, subprocess, uuid, time, os, glob
from datetime import datetime as dt, timedelta as td from datetime import datetime as dt, timedelta as td
from sharedfunctions import getfolderid, callrestapi from sharedfunctions import getfolderid, callrestapi, getpath
# get python version # get python version
version=int(str(sys.version_info[0])) version=int(str(sys.version_info[0]))
# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION # CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
clidir='/opt/sas/viya/home/bin/' clidir='/opt/sas/viya/home/bin/'
# get input parameters # get input parameters
parser = argparse.ArgumentParser(description="Export the complete Viya folder tree") parser = argparse.ArgumentParser(description="Export Viya Reports each to its own unique transfer package")
parser.add_argument("-d","--directory", help="Directory for Export",required='True') parser.add_argument("-d","--directory", help="Directory to store report packages",required='True')
parser.add_argument("-q","--quiet", help="Suppress the are you sure prompt.", action='store_true') parser.add_argument("-q","--quiet", help="Suppress the are you sure prompt.", action='store_true')
parser.add_argument("-c","--changeddays", help="Reports changed in the how many days",default='1') parser.add_argument("-c","--changeddays", help="Reports changed in the how many days (defaults to 1 day)?",default='1')
parser.add_argument("-m","--modifiedby", help="Last modified id equals",default=None) parser.add_argument("-m","--modifiedby", help="Last modified id equals?",default=None)
parser.add_argument("-n","--name", help="Name contains",default=None) parser.add_argument("-n","--name", help="Name contains?",default=None)
parser.add_argument("-f","--folderpath", help="Folder Path starts with?",default="/")
args= parser.parse_args() args= parser.parse_args()
basedir=args.directory basedir=args.directory
@ -60,7 +63,7 @@ quietmode=args.quiet
changeddays=args.changeddays changeddays=args.changeddays
modby=args.modifiedby modby=args.modifiedby
nameval=args.name nameval=args.name
folderpath=args.folderpath
# calculate time period for files # calculate time period for files
now=dt.today()-td(days=int(changeddays)) now=dt.today()-td(days=int(changeddays))
@ -79,7 +82,6 @@ if modby!=None: filtercond.append("eq(modifiedBy,"+modby+")")
# add the start and end and comma delimit the filter # add the start and end and comma delimit the filter
delimiter = ',' delimiter = ','
completefilter = 'and('+delimiter.join(filtercond)+')' completefilter = 'and('+delimiter.join(filtercond)+')'
print(completefilter)
# prompt if directory exists because existing json files are deleted # prompt if directory exists because existing json files are deleted
if os.path.exists(basedir): if os.path.exists(basedir):
@ -90,11 +92,11 @@ if os.path.exists(basedir):
if version > 2: if version > 2:
areyousure=input("The folder exists any existing json files in it will be deleted. Continue? (Y)") areyousure=input("The folder exists any existing json files in it will be deleted. Continue? (Y)")
else: else:
areyousure=raw_input("The folder already exists any existing json files in it will be deleted. Continue? (Y)") areyousure=raw_input("The folder already exists any existing json files in it will be deleted. Continue? (Y)")
else: else:
areyousure="Y" areyousure="Y"
else: areyousure="Y" else: areyousure="Y"
# prompt is Y if user selected Y, its a new directory, or user selected quiet mode # prompt is Y if user selected Y, its a new directory, or user selected quiet mode
if areyousure.upper() =='Y': if areyousure.upper() =='Y':
@ -103,7 +105,7 @@ if areyousure.upper() =='Y':
# create directory if it doesn't exist # create directory if it doesn't exist
if not os.path.exists(path): os.makedirs(path) if not os.path.exists(path): os.makedirs(path)
else: else:
filelist=glob.glob(path+"/*.json") filelist=glob.glob(path+"/*.json")
for file in filelist: for file in filelist:
os.remove(file) os.remove(file)
@ -111,43 +113,57 @@ if areyousure.upper() =='Y':
# retrieve all reports in the system # retrieve all reports in the system
reqtype='get' reqtype='get'
reqval='/reports/reports?filter='+completefilter+'&limit=10000' reqval='/reports/reports?filter='+completefilter+'&limit=10000'
resultdata=callrestapi(reqval,reqtype) resultdata=callrestapi(reqval,reqtype)
# loop root reports # loop root reports
if 'items' in resultdata: if 'items' in resultdata:
total_items=resultdata['count'] total_items=resultdata['count']
returned_items=len(resultdata['items']) returned_items=len(resultdata['items'])
if total_items == 0: print("Note: No items returned.") if total_items == 0: print("Note: No items returned.")
else: else:
# export each folder and download the package file to the directory # export each folder and download the package file to the directory
for i in range(0,returned_items):
reports_exported=0
for i in range(0,returned_items):
id=resultdata['items'][i]["id"] id=resultdata['items'][i]["id"]
package_name=str(uuid.uuid1())
json_name=resultdata['items'][i]["name"].replace(" ","")+'_'+str(i) path_to_report=getpath("/reports/reports/"+id)
json_name=json_name.replace("(","_") if path_to_report.startswith(folderpath):
json_name=json_name.replace(")","_")
reports_exported=reports_exported+1
command=clidir+'sas-admin transfer export -u /reports/reports/'+id+' --name "'+package_name+'"'
print(command) path_to_report=path_to_report.replace("/","_")
subprocess.call(command, shell=True)
package_name=str(uuid.uuid1())
reqval='/transfer/packages?filter=eq(name,"'+package_name+'")' json_name=path_to_report+resultdata['items'][i]["name"].replace(" ","")+'_'+str(i)
package_info=callrestapi(reqval,reqtype)
json_name=json_name.replace("(","_")
package_id=package_info['items'][0]['id'] json_name=json_name.replace(")","_")
json_name=json_name.replace(" ","-")
completefile=os.path.join(path,json_name+'.json')
command=clidir+'sas-admin transfer download --file '+completefile+' --id '+package_id command=clidir+'sas-admin transfer export -u /reports/reports/'+id+' --name "'+package_name+'"'
print(command) print(command)
subprocess.call(command, shell=True) subprocess.call(command, shell=True)
print("NOTE: Viya reports exported to json files in "+path) reqval='/transfer/packages?filter=eq(name,"'+package_name+'")'
package_info=callrestapi(reqval,reqtype)
package_id=package_info['items'][0]['id']
completefile=os.path.join(path,json_name+'.json')
command=clidir+'sas-admin transfer download --file '+completefile+' --id '+package_id
print(command)
subprocess.call(command, shell=True)
print("NOTE: "+str(reports_exported)+" Viya report(s) exported to json files in "+path)
else: else:
print("NOTE: Operation cancelled") print("NOTE: Operation cancelled")

Loading…
Cancel
Save