Skip to content

Commit

Permalink
Version 1.1.0 new and fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@trianglesis
trianglesis committed Sep 25, 2017
1 parent 24b9cd4 commit 4eed7fd
Show file tree
Hide file tree
Showing 10 changed files with 447 additions and 211 deletions.
4 changes: 3 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,6 @@ bmc_tplpre/DEV_TEST/TEST_BMC_TPL_Ide_All.py
bmc_tplpre/DEV_TEST/test_executor.py
bmc_tplpre/DEV_TEST/tests_init.json
bmc_tplpre/DEV_TEST/test_executor.py
bmc_tplpre/DEV_TEST/test_executor.py
bmc_tplpre/DEV_TEST/test_executor.py
bmc_tplpre/DEV_TEST/dev_progressbar.py
bmc_tplpre/DEV_TEST/progress_examples.py
8 changes: 8 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
## 1.1.0 2017-09-25
- Fix bugs with solo uploading for Customer\Dev logics.
- Added progress bars for long processes.
- Added new -wipe_tku arg and function - for TOTAL FORCED wipe TKU!
- Better show and prognoses ETA for related tests run.
- Add more informative exceptions for unsupported args and modes.


## 1.0.1 2017-09-21
- Fix bug with no uploading.
- Remove unused code
Expand Down
5 changes: 5 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ NOTE: syntax tests (require 3rd party [module tplint](https://communities.bmc.co
- pip install **paramiko**, **progressbar2**
- use arguments;

**Required**:
- To use ADDM over SSH commands: [paramiko](https://github.com/paramiko/paramiko)

Optional:
- To print nice and fancy progress bars for long processes: [progressbar](https://github.com/WoLpH/python-progressbar)

## Usage: ##

Expand Down
15 changes: 13 additions & 2 deletions bmc_tplpre/check.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
common = parser.add_argument_group("Common options")
developer = parser.add_argument_group("Developer options")

developer.add_argument("-wipe_tku",
action="store_true",
help="Totally wipe knowledge update with "
"tw_pattern_management --remove-all --force.")
developer.add_argument("-usual_import",
action="store_true",
help="Option imports patterns which only imported in currently opened pattern "
Expand Down Expand Up @@ -109,7 +113,7 @@
help="Please set log level") # info, quiet, warning, debug, output, error
common.add_argument('--version',
action='version',
version='%(prog)s 1.0.1')
version='%(prog)s 1.1.0')

known_args, extra_args = parser.parse_known_args()
# print("Known args: "+str(known_args))
Expand Down Expand Up @@ -169,6 +173,13 @@
log.debug("ZIP:\t\tzip_files_f")
zip_files_f()

# Wiping TKU!:
if callable(conditional_functions['wipe_tku_f']):
wipe_tku_f = conditional_functions['wipe_tku_f']
if wipe_tku_f:
log.debug("UPLOAD:\t\twipe_tku_f")
wipe_tku_f()

# Executing pattern upload:
if callable(conditional_functions['upload_f']):
upload_f = conditional_functions['upload_f']
Expand All @@ -189,7 +200,7 @@
scan_f = conditional_functions['scan_f']
if scan_f:
log.debug("SCAN:\t\tscan_f")
# scan_f()
scan_f()

if callable(conditional_functions['test_executor_f']):
test_executor = conditional_functions['test_executor_f']
Expand Down
138 changes: 109 additions & 29 deletions bmc_tplpre/check/global_logic.py
View file

Large diffs are not rendered by default.

8 changes: 7 additions & 1 deletion bmc_tplpre/check/imports.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,9 +556,15 @@ def _del_old_imports(self, path):
log.debug("Step 4. Wiping 'imports' folder before add new imports in it. "+str(path))
shutil.rmtree(path, onerror=self._del_rw)
# shutil.rmtree(path)
except:
except TypeError as e:
log.warning("Step 4.1 This folder exist but program have no permission to remove it. "
"Please check path and permissions and 'AR' attribute in file.")
log.error(e)
raise
except PermissionError as e:
log.warning("Step 4.1 This folder exist but program have no permission to remove it. "
"Please check path and permissions and 'AR' attribute in file.")
log.error(e)
raise

@staticmethod
Expand Down
145 changes: 67 additions & 78 deletions bmc_tplpre/check/local_logic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,56 +228,60 @@ def addm_compose_paths(dev_vm_path, pattern_folder):
return working_dir_virt, pattern_test_virt

@staticmethod
def make_zip(path, module_name):
def make_zip(path, module_name, mode_single=None):
"""
Zip pattern files in path.
:param mode_single: Zip files in folder or only one file.
:param module_name: Name of pattern or its folder to create zip with its name.
:param path: Path where tpl files are ready to be zipped.
:return: zip path to ready zip file.
"""

norm_path = os.path.normpath(path+os.sep)
path = norm_path+os.sep
if not mode_single:
norm_path = os.path.normpath(path+os.sep)
path = norm_path+os.sep
log.debug("ZIP: Making zip of all .tpl in folder:" + norm_path)
zip_filename = module_name + '.zip'
zip_path = path + zip_filename

log.debug("ZIP: Making zip of all .tpl in folder:" + norm_path)
try:
patterns_zip = zipfile.ZipFile(zip_path, 'w')
log.debug("ZIP: zip_filename:" + zip_filename)

zip_filename = module_name + '.zip'
for foldername, subfolders, filenames in os.walk(norm_path):
for filename in filenames:
if filename != zip_filename:

zip_path = path + zip_filename
patterns_zip.write(os.path.join(norm_path, filename), arcname=filename)
log.debug("ZIP: Adding pattern:" + filename)

try:
patterns_zip = zipfile.ZipFile(zip_path, 'w')
log.debug("ZIP: zip_filename:" + zip_filename)

for foldername, subfolders, filenames in os.walk(norm_path):
for filename in filenames:
if filename != zip_filename:

# check if file mod date is recent:
# TODO: Check current file or folder mod time?
# file_time = os.path.getmtime(path+module_name)
# now = datetime.datetime.now()
# ago = now - datetime.timedelta(minutes=15)
# file_time_stamp = datetime.datetime.fromtimestamp(file_time)
#
# if file_time_stamp < ago:
#
# log.warning("ZIP: TPLPreprocessor result files looks like older that 15 min. "
# "Please check: " + str(file_time_stamp))
#
# if file_time_stamp > ago:
#
# log.debug("ZIP: TPLPreprocessor result files are recent: " + str(file_time_stamp))

patterns_zip.write(os.path.join(norm_path, filename), arcname=filename)
log.debug("ZIP: Adding pattern:" + filename)
patterns_zip.close()
log.debug("ZIP: zip_path: " + zip_path)
return zip_path

except FileNotFoundError as e:
log.error("Patterns cannot be zipped because file path does not exist: "+str(e))
return False
patterns_zip.close()
log.debug("ZIP: zip_path: " + zip_path)
return zip_path

except FileNotFoundError as e:
log.error("Patterns cannot be zipped because file path does not exist: "+str(e))
return False
else:
norm_path = os.path.normpath(path)
tpl_path = norm_path + '.tpl'
zip_path = norm_path + '.zip'
log.debug("ZIP: Zipping single file: " + tpl_path)
zip_filename = module_name + '.zip'

try:
with zipfile.ZipFile(zip_path, 'w') as zip_pattern:
zip_pattern.write(tpl_path, arcname=module_name+'.tpl')

log.debug("ZIP: zip_filename:" + zip_filename)
log.debug("ZIP: Adding pattern:" + tpl_path)
log.debug("ZIP: zip_path: " + zip_path)

return zip_path

except FileNotFoundError as e:
log.error("Patterns cannot be zipped because file path does not exist: "+str(e))
return False

def file_path_decisions(self, full_file_path):
"""
Expand Down Expand Up @@ -389,21 +393,27 @@ def file_path_decisions(self, full_file_path):

# Check if this is a tpl file from: PatternFolder\tpl110\PatternName.tpl
elif re.match('tpl', file_ext):
log.debug("File extension matched tpl pattern. DEV in progress...")
log.info("File extension matched tpl pattern. DEV in progress...")

path_parse = self.tpl_path_parse_re.match(full_file_path)
pattern_folder = path_parse.group('pattern_folder')
file_name = path_parse.group('file_name')
file_ext = path_parse.group('file_ext')
tpl_folder = path_parse.group('tpl_folder')

# pattern working dir, where pattern file is really lies:
# 'd:\\perforce\\addm\\tkn_main\\tku_patterns\\CORE\\BMCRemedyARSystemtp\\l114\\check.log
working_dir = tku_patterns_t+os.sep+pattern_lib+os.sep+pattern_folder+os.sep+tpl_folder
# Making prognosable place to test.py - but check if exist in global logic mod
pattern_test_t = working_dir+os.sep+'tests'+os.sep+'test.py'

args_dict = dict(
environment_condition = 'developer_tpl',
workspace = workspace,
pattern_folder = pattern_folder,
file_name = file_name,
file_ext = file_ext,
working_dir = tpl_folder,
working_dir = working_dir,
full_path = full_file_path,
tkn_main_t = tkn_main_t,
tku_patterns_t = tku_patterns_t,
Expand All @@ -417,7 +427,8 @@ def file_path_decisions(self, full_file_path):
MIDDLEWAREDETAILS_t = MIDDLEWAREDETAILS_t,
STORAGE_t = STORAGE_t,
SYSTEM_t = SYSTEM_t,
tkn_sandbox_t = tkn_sandbox_t
tkn_sandbox_t = tkn_sandbox_t,
pattern_test_t = pattern_test_t
)
log.debug("Arguments from file path: " + str(args_dict))
log.debug("TPL: File extension mach .tpl and dev_path_check is found, "
Expand All @@ -427,7 +438,6 @@ def file_path_decisions(self, full_file_path):
# Check if this is a dml file from: ..\tests\dml\DML_DATA.dml
elif re.match('dml', file_ext):
log.debug("File extension matched dml pattern. DEV in progress...")

# TODO: Add pattern folder based on regex path to dml
log.debug("This is DML file.")
args_dict = dict(
Expand Down Expand Up @@ -455,7 +465,8 @@ def file_path_decisions(self, full_file_path):
log.debug("Arguments from file path: " + str(args_dict))
log.debug("DML: File extension mach .dml and dev_path_check is found, "
"options will be set based on it's path.")
return args_dict
# return args_dict
raise Exception("DML Files is not supported yet.")

# Check if this is a model file from: \tests\actuals\SI_MODEL.model
elif re.match('model', file_ext):
Expand Down Expand Up @@ -488,7 +499,8 @@ def file_path_decisions(self, full_file_path):
log.debug("Arguments from file path: " + str(args_dict))
log.debug("MODEL: File extension mach .model and dev_path_check is found, "
"options will be set based on it's path.")
return args_dict
# return args_dict
raise Exception("MODEL Files is not supported yet.")

# Check if this is a py file from: ..\tests\test.py
elif re.match('py', file_ext):
Expand Down Expand Up @@ -520,16 +532,16 @@ def file_path_decisions(self, full_file_path):
log.debug("Arguments from file path: " + str(args_dict))
log.debug("PY: File extension mach .py and dev_path_check is found, "
"options will be set based on it's path.")
return args_dict
# return args_dict
raise Exception("PY Files is not supported yet.")

# If this file has an extension I do not support:
else:
log.warning("FILE: Did not match any file extension "
"I can use: 'tpl', 'tplpre', 'dml', 'model', 'test.py'")
log.debug("Path matched and parsed.")
raise Exception("FILE: Did not match any file extension "
"I can use: 'tpl', 'tplpre', 'dml', 'model', 'test.py'")
else:
log.warning("Did not match TKU DEV pattern path tree! Will use another way to parse.")
log.debug("I expect path to file: d:\\P4\\addm\\tkn_main\\tku_patterns\\..\\..\\FileName.Ext")
raise Exception("Did not match TKU DEV pattern path tree! Will use another way to parse."
"I expect path to file: d:\\P4\\addm\\tkn_main\\tku_patterns\\..\\..\\FileName.Ext")

elif tku_path_check:
log.info("Found Technology-Knowledge-Update path.")
Expand Down Expand Up @@ -572,30 +584,6 @@ def file_path_decisions(self, full_file_path):
file_name = tku_pack_parse.group('file_name')
file_ext = tku_pack_parse.group('file_ext')

# print("working_dir: '{}' "
# "tku_update_path: '{}' "
# "workspace: '{}' "
# "tku_package: '{}' "
# "tku_package_year: '{}' "
# "tku_package_month: '{}' "
# "tku_package_day: '{}' "
# "tku_package_ADDM_ver: '{}' "
# "tku_package_name: '{}' "
# "pattern_folder: '{}' "
# "file_name: '{}' "
# "file_ext: '{}' ". format(working_dir,
# tku_update_path,
# workspace,
# tku_package,
# tku_package_year,
# tku_package_month,
# tku_package_day,
# tku_package_ADDM_ver,
# tku_package_name,
# pattern_folder,
# file_name,
# file_ext))

log.debug("I found following TKU Package details:"
" tku_update_path: " + str(tku_update_path) +
" tku_package: " + str(tku_package) +
Expand Down Expand Up @@ -832,11 +820,12 @@ def file_path_decisions(self, full_file_path):
log.warning("This path did not match any suitable pattern and probably not a tpl file"
" Or path has superfluous symbols or spaces")
else:
log.debug("FILE: Cannot match file path for alone pattern. "
"I expect: d:\\Something\\SomePattern.(tpl|tplpre)")
raise FileNotFoundError("FILE: Cannot match file path for alone pattern. "
"I expect: d:\\Something\\SomePattern.(tpl|tplpre)")

else:
log.critical("No '-full_path' argument was set. Or this path is not exist!")
# log.critical("No '-full_path' argument was set. Or this path is not exist!")
raise FileNotFoundError("No '-full_path' argument was set. Or this path is not exist!")

def workspace_find_p4_env(self):
"""
Expand Down
15 changes: 11 additions & 4 deletions bmc_tplpre/check/parse_args.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,10 +286,12 @@ def oper_mode(known_args):
"""

oper_args_set = dict(
imports = dict(recursive_imports=False, usual_imports=False, read_test=False
),
tests = dict(related_tests=False, run_test=False
)
imports = dict(recursive_imports=False,
usual_imports=False,
read_test=False),
tests = dict(related_tests=False,
run_test=False),
tku_operations = dict(wipe_tku=False)
)

if known_args.read_test and known_args.recursive_import:
Expand Down Expand Up @@ -327,4 +329,9 @@ def oper_mode(known_args):
oper_args_set['tests'] = False
oper_args_set['imports'] = False

if known_args.wipe_tku:
oper_args_set['tku_operations']['wipe_tku'] = True
else:
oper_args_set['tku_operations']['wipe_tku'] = False

return oper_args_set
Loading

0 comments on commit 4eed7fd

Please sign in to comment.