add datefilter function and use it (#118)

* add datefilter function and use it

* remove debug

application.properties

@@ -1,2 +1,2 @@
 sascli.location=/opt/sas/viya/home/bin/
-sascli.executable=sas-admin
+sascli.executable=sas-viya

archivefiles.py

@@ -31,7 +31,7 @@
 #
 
 import argparse , datetime, os, time, json, sys
-from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris
+from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris,createdatefilter
 from datetime import datetime as dt, timedelta as td
 
 # get python version
@@ -83,11 +83,8 @@ if dodelete:
 
    if areyousure !='Y': dodelete=False
 
-
 # calculate time period for files
-now=dt.today()-td(days=int(daysolder))
+datefilter=createdatefilter(olderoryounger='older',datevar='creationTimeStamp',days=daysolder)
-subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
-datefilter="le(creationTimeStamp,"+subset_date+")"
 
 # create a list for filter conditions
 filtercond=[]

deletetransferpackages.py

@@ -23,7 +23,7 @@
 import argparse , datetime, sys, json
 
 from requests import delete
-from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris
+from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris,createdatefilter
 from datetime import datetime as dt, timedelta as td
 
 # setup command-line arguements. In this block which is common to all the tools you setup what parameters
@@ -57,13 +57,8 @@ quietmode=args.quiet
 
 packagefile_result=None
 
-# calculate time period for files
+# build the date filter
-now=dt.today()-td(days=int(days))
+datefilter=createdatefilter(olderoryounger=olderoryounger,datevar='creationTimeStamp',days=days)
-subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
-
-if olderoryounger=='older':
-  datefilter="le(creationTimeStamp,"+subset_date+")"
-else: datefilter="ge(creationTimeStamp,"+subset_date+")"
 
 # create a list for filter conditions
 filtercond=[]

listfiles.py

@@ -33,7 +33,7 @@
 #
 
 import argparse , datetime, sys
-from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris
+from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris,createdatefilter
 from datetime import datetime as dt, timedelta as td
 
 # setup command-line arguements. In this block which is common to all the tools you setup what parameters
@@ -78,12 +78,7 @@ if puri !=None and pfolder !=None:
    sys.exit()
 
 # calculate time period for files
-now=dt.today()-td(days=int(days))
+datefilter=createdatefilter(olderoryounger=olderoryounger,datevar='creationTimeStamp',days=days)
-subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
-
-if olderoryounger=='older':
-  datefilter="le(creationTimeStamp,"+subset_date+")"
-else: datefilter="ge(creationTimeStamp,"+subset_date+")"
 
 # create a list for filter conditions
 filtercond=[]

listmodelobjects.py

@@ -14,7 +14,7 @@
 #
 
 import argparse , datetime, sys
-from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris
+from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris,createdatefilter
 from datetime import datetime as dt, timedelta as td
 
 # setup command-line arguments. In this block, which is common to all the tools, you setup what parameters
@@ -29,6 +29,8 @@ parser.add_argument("-d","--days", help="List files older than this number of da
 parser.add_argument("-m","--modifiedby", help="Last modified id equals",default=None)
 parser.add_argument("-s","--sortby", help="Sort the output descending by this field",default='name')
 parser.add_argument("-o","--output", help="Output Style", choices=['csv','json','simple'],default='csv')
+parser.add_argument("--debug", action='store_true', help="Debug")
+
 
 args = parser.parse_args()
 output_style=args.output
@@ -37,11 +39,14 @@ modby=args.modifiedby
 sortby=args.sortby
 nameval=args.name
 typeval=args.type
+debug=args.debug
 
 # calculate time period for models
-now=dt.today()-td(days=int(daysolder))
+# now=dt.today()-td(days=int(daysolder))
-subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
+# subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
-datefilter="le(creationTimeStamp,"+subset_date+")"
+# datefilter="le(creationTimeStamp,"+subset_date+")"
+
+datefilter=createdatefilter(olderoryounger='older',datevar='creationTimeStamp',days=daysolder)
 
 # create a list for filter conditions
 filtercond=[]
@@ -57,15 +62,21 @@ completefilter = 'and('+delimiter.join(filtercond)+')'
 
 # prepare the request according to content type
 reqtype='get'
-if typeval=='model': reqval="/modelRepository/models?"+completefilter+"&sortBy="+sortby
+
-if typeval=='project': reqval="/modelRepository/projects?"+completefilter+"&sortBy="+sortby
+if typeval=='model': reqval="/modelRepository/models?filter="+completefilter+"&sortBy="+sortby
-if typeval=='repository': reqval="/modelRepository/repositories?"+completefilter+"&sortBy="+sortby
+if typeval=='project': reqval="/modelRepository/projects?filter="+completefilter+"&sortBy="+sortby
+if typeval=='repository': reqval="/modelRepository/repositories?filter="+completefilter+"&sortBy="+sortby
+
+if debug: print(reqval)
 
 # Make call, and process & print results
 files_result_json=callrestapi(reqval,reqtype)
-print("-------------------------------------")
+
-print("Listing "+typeval+" objects")
+# remove not consistent with other tools
-print("-------------------------------------")
+# print("-------------------------------------")
+# print("Listing "+typeval+" objects")
+# print("-------------------------------------")
+
 printresult(files_result_json,output_style)
 
 

listtransferpackages.py

@@ -21,7 +21,7 @@
 #
 
 import argparse , datetime, sys
-from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris
+from sharedfunctions import callrestapi,printresult,getfolderid,getidsanduris,createdatefilter
 from datetime import datetime as dt, timedelta as td
 
 parser = argparse.ArgumentParser()
@@ -50,14 +50,8 @@ olderoryounger=args.olderoryounger
 
 packagefile_resultn=None
 
-# calculate time period for files
+# build the date filter
-# calculate time period for files
+datefilter=createdatefilter(olderoryounger=olderoryounger,datevar='creationTimeStamp',days=days)
-now=dt.today()-td(days=int(days))
-subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
-
-if olderoryounger=='older':
-  datefilter="le(creationTimeStamp,"+subset_date+")"
-else: datefilter="ge(creationTimeStamp,"+subset_date+")"
 
 # create a list for filter conditions
 filtercond=[]

sharedfunctions.py

@@ -38,6 +38,7 @@
 #  20Feb2022 Support patch
 #  28Feb2022 Added functionality to callrestapi optionally pass in etags, and to request they be returned, for API endpoints that use them
 #  08Sep2022 Catch Unicode error in get_valid_filename and remove string function if it happens
+#  12OCT2022 Build date filter function
 #
 # Copyright © 2018, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 #
@@ -65,6 +66,7 @@ import os
 import collections
 import inspect
 import re
+from datetime import datetime as dt, timedelta as td
 
 pp = pprint.PrettyPrinter(indent=4)
 
@@ -747,4 +749,19 @@ def getapplicationproperties():
     myparams=dict(line.strip().split('=') for line in open(prop_file) if line[0].isalpha())
     return myparams
 
+# build a date filter for the REST filter
 
+def createdatefilter(days=0,datevar='creationTimeStamp',olderoryounger='older'):
+    
+    # what date is the filter based on
+    thedate=dt.today()-td(days=int(days))
+
+    # set the timestamp to be at the end of the day for younger and the begining for older
+    if olderoryounger=='older':
+       subset_date=thedate.replace(hour=23, minute=59, second=59, microsecond=999999).strftime("%Y-%m-%dT%H:%M:%S")
+       datefilter="le("+datevar+","+subset_date+")"
+    else:
+       subset_date=thedate.replace(hour=00, minute=00, second=00, microsecond=999999).strftime("%Y-%m-%dT%H:%M:%S")
+       datefilter="ge("+datevar+","+subset_date+")"
+   
+    return datefilter

snapshotreports.py

@@ -43,7 +43,7 @@
 import re
 import argparse, sys, subprocess, uuid, time, os, glob
 from datetime import datetime as dt, timedelta as td
-from sharedfunctions import getfolderid, callrestapi, getpath, getapplicationproperties, get_valid_filename
+from sharedfunctions import getfolderid, callrestapi, getpath, getapplicationproperties, get_valid_filename, createdatefilter
 
 
 # get python version
@@ -81,9 +81,11 @@ nameval=args.name
 folderpath=args.folderpath
 
 # calculate time period for files
-now=dt.today()-td(days=int(changeddays))
+# now=dt.today()-td(days=int(changeddays))
-subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
+# subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
-datefilter="ge(modifiedTimeStamp,"+subset_date+")"
+# datefilter="ge(modifiedTimeStamp,"+subset_date+")"
+
+datefilter=createdatefilter(olderoryounger='younger',datevar='modifiedTimeStamp',days=changeddays)
 
 # create a list for filter conditions
 filtercond=[]