This is for reference and has been html formatted, download from the links on the main page
#!/bin/env /usr/bin/python
# change above line to local site needs
#
# *** procrc.py - Copyright (C) 2015 Frank Spaniak All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# procrc.py - display the CRCs of source files of a compiled progress program
# based on procrc.c from Grant P. Maizels. credits go to his work.
#
# Python port and maintainer:
# Frank Spaniak (python.procrc at gmail dot com, http://procrc.spaniak.org)
#
# with appreciation for the help from:
# Progress Tools (http://progress-tools.x10.mx)
#
# Version history
# 1.0 fjs Initial release
#
#
#
# TODO: Python 3 port
#
import sys
import os
import datetime
import struct
import array
import optparse
import textwrap
from optparse import OptionParser
#if sys.version_info < (3,0):
# from __builtin__ import True
# Copyright and credits do not remove
copyright = "***procrc.py - Copyright (C) 2015 Frank Spaniak, see --help"
program_credits=""" Display file and crc information for Progress dotr file(s)
procrc.py - Copyright (C) 2015 Frank Spaniak All rights reserved
(python.procrc@gmail.com, http://procrc.spaniak.org/)
Original Work and credit to Grant P. Maizels
With thanks to Progress Tools (http://progress-tools.x10.mx)
"""
READ_FROM_FILES = True
READ_TO_FILES = False
errors_encountered = None
testing_mode = False #
debug_it = False #
file_list = []
do_export = False
do_sort = False
do_tab = False
do_headers_only = False
do_compare = False
comp_result = {} # dictionary to hold compare function results
from_dir = None
to_dir = None
rcode_version = 0
header1_size = 0
text_segment_size = 0
pointer = 0
byteswapped = False
# ----- right out of procrc.c
O_MAGIC = 0
O_TS = 4
# signature size
O_HDR1 = 8 # v9+ but zero previously - so there should be no compatibility problems
O_HDR11 = 56 # v11
O_RCVER = int(0xE) # 14
O_SEGT_SZ = int(0x1E) # 30
HDR_SZ = int(0x44) # 68
#
# constants
# SEGMENT LIST
I_SEG_AD = 0
DEBUG_SEG_AD_V9 = int(0x18)
DEBUG_SEG_AD_V10 = int(0x24)
DEBUG_SEG_AD_V11 = int(0xC) # Progress Tools Thanks!
# crc offset
O_PROG_CRC_V9 = int(0x46) # 70
O_PROG_CRC_V10 = int(0x6E) # 110
O_PROG_CRC_V11 = int(0xA4) # 164
# DEBUG HEADER
O_SRC_CNT_V9 = 2
O_SRC_CNT_V10 = int(0x14)
O_SRC_CNT_V11 = int(0x14)
O_SRC_LST = 4
# DEBUG SRC LIST
O_DEBSRC_NEXT = 0
O_DEBSRC_CRC_V9 = 2
O_DEBSRC_CRC_V10 = 4
O_DEBSRC_CRC_V11 = 4
#O_DEBSRC_PPSZ_V9 = 4
#O_DEBSRC_PPSZ_V10 = int(0xA)
#O_DEBSRC_PPSZ_V11 = int(0xA)
O_DEBSRC_FILE_V9 = 9
O_DEBSRC_FILE_V10 = int(0xB)
O_DEBSRC_FILE_V11 = int(0xB)
# to fix optparse formatting so newlines work better
class IndentedHelpFormatterWithNL(optparse.IndentedHelpFormatter):
def format_description(self, description):
if not description: return ""
desc_width = self.width - self.current_indent
indent = " "*self.current_indent
# the above is still the same
bits = description.split('\n')
formatted_bits = [
textwrap.fill(bit,
desc_width,
initial_indent=indent,
subsequent_indent=indent) for bit in bits]
result = "\n".join(formatted_bits) + "\n"
return result
def format_option(self, option):
# The help for each option consists of two parts:
# * the opt strings and metavars
# eg. ("-x", or "-fFILENAME, --file=FILENAME")
# * the user-supplied help string
# eg. ("turn on expert mode", "read data from FILENAME")
#
# If possible, we write both of these on the same line:
# -x turn on expert mode
#
# But if the opt string list is too long, we put the help
# string on a second line, indented to the same column it would
# start in if it fit on the first line.
# -fFILENAME, --file=FILENAME
# read data from FILENAME
result = []
opts = self.option_strings[option]
opt_width = self.help_position - self.current_indent - 2
if len(opts) > opt_width:
opts = "%*s%s\n" % (self.current_indent, "", opts)
indent_first = self.help_position
else: # start help on same line as opts
opts = "%*s%-*s " % (self.current_indent, "", opt_width, opts)
indent_first = 0
result.append(opts)
if option.help:
help_text = self.expand_default(option)
# Everything is the same up through here
help_lines = []
for para in help_text.split("\n"):
help_lines.extend(textwrap.wrap(para, self.help_width))
# Everything is the same after here
result.append("%*s%s\n" % (
indent_first, "", help_lines[0]))
result.extend(["%*s%s\n" % (self.help_position, "", line)
for line in help_lines[1:]])
elif opts[-1] != "\n":
result.append("\n")
return "".join(result)
#
# debugging and helpers
#
def print_bytes(addr, byteswapped):
laddr = addr
#if debug_it: print(" %s print bytes at at address: %s (0x%lx) %s" % (m, laddr, laddr, binascii.hexlify(memoryview(bytearray(p[laddr:laddr+4]))) ))
a = [0]*16
if byteswapped:
for i, x in reversed(list(enumerate(p[laddr:laddr+16]))):
a[i] = x
else:
for i, x in enumerate(p[laddr:laddr+16]):
a[i] = x
z = " ".join("{:02x}".format(x) for x in a)
b = " ".join("{:04x}".format(laddr))
if debug_it: print " print bytes at (",laddr,") " ,b," :", z
def print_bytes_big(m, addr, byteswapped):
laddr = addr
xlen = 16*16
for v in range(laddr, laddr + xlen, 16):
print_bytes(m, v, byteswapped)
# for progress export function
def doq(instring):
return instring.replace('\"', '\"\"')
def enq(instring):
return '\"' + instring + '\"'
def edq(instring):
return enq(doq(instring))
# used by sort option
def sort_key(item):
return item[0]
# bit flipping routines
def get_double(addr, byteswapped):
laddr = pointer + addr
b = [0]*8 # init array to zero
if byteswapped:
for i, x in reversed(list(enumerate(p[laddr:laddr+8]))):
b[i] = x
else:
for i, x in enumerate(p[laddr:laddr + 8]):
b[i] = x
double_val = struct.unpack("<q", "".join(map(chr, bytearray(b))))
return double_val
def get_long(addr, byteswapped):
laddr = pointer + addr
if byteswapped:
return ((p[laddr+3] * 256 * 65536) +
(p[laddr+2] * 65536) +
(p[laddr+1] * 256) +
p[laddr])
else:
return ((p[laddr] * 256 * 65536) +
(p[laddr+1] * 65536) +
(p[laddr+2] * 256) +
p[laddr+3])
def get_short(addr, byteswapped):
laddr = pointer + addr
if byteswapped:
return ((p[laddr+1] * 256) + p[laddr])
else:
return ((p[laddr] * 256) + p[laddr+1])
def get_byte(addr, byteswapped):
laddr = pointer + addr
return p[laddr]
def get_byte_a(addr): # absolute no offset
return p[addr]
def get_string_a(addr, s_length):
x = "".join( chr( val ) for val in p[addr: addr + s_length] )
return x
def null_filter(still, int_char):
if not still:
return False
if int_char == 0:
still = False
return False
return True
def get_string_terminated(addr):
keep_going = True
max_len = 18
x = []
for val in p[addr: addr + max_len]:
keep_going = null_filter(keep_going, val)
if keep_going:
x.append(chr(val))
else:
break
return "".join(x)
# the pointer is added in, in the subordinate routine
def get_var_size(addr, the_size, byteswapped):
if the_size == 1:
return get_byte(addr, byteswapped)
elif the_size == 2:
return get_short(addr, byteswapped)
elif the_size == 4:
return get_long(addr, byteswapped)
elif the_size == 8:
return get_double(addr, byteswapped)
# Main Program
def get_procrc(file_to_open, primary_list):
global p, byteswapped, pointer
rcode_version = 0
header1_size = 0
text_segment_size = 0
pointer = 0
# see if file has progress r-code signature
if get_long(O_MAGIC, byteswapped) != 0x56ced309:
if get_long(O_MAGIC, byteswapped) == 0x09d3ce56:
byteswapped = True
else:
if not do_export and not do_compare:
print("Not a progress R-code file\n")
return
rcode_version = get_short(O_RCVER, byteswapped) # works for all
#is_64bits = False
#if (rcode_version & 0x4000) != 0:
# is_64bits = True
if (rcode_version & 0x3FFF) >= 1100:
if debug_it: print(" version 11 code detected")
version = 11
o_src_cnt = O_SRC_CNT_V11 # 28
o_debsrc_crc = O_DEBSRC_CRC_V11 # 4
o_debsrc_file = O_DEBSRC_FILE_V11 # 11
o_prog_crc = O_PROG_CRC_V11 # 110
sz_debsrc_next = 4
sz_src_lst = 4
elif (rcode_version & 0x3FFF) >= 1000:
if debug_it: print(" version 10 code detected")
version = 10
o_src_cnt = O_SRC_CNT_V10
o_debsrc_crc = O_DEBSRC_CRC_V10
o_debsrc_file = O_DEBSRC_FILE_V10
o_prog_crc = O_PROG_CRC_V10
sz_debsrc_next = 4
sz_src_lst = 4
else:
if debug_it: print(" version 9 code detected")
version = 9
o_src_cnt = O_SRC_CNT_V9
o_debsrc_crc = O_DEBSRC_CRC_V9
o_debsrc_file = O_DEBSRC_FILE_V9
o_prog_crc = O_PROG_CRC_V9
sz_debsrc_next = 2
sz_src_lst = 2
if (version == 11):
debug_seg_ad = DEBUG_SEG_AD_V11
else:
if (rcode_version & 0x3FFF) >= 909:
debug_seg_ad = DEBUG_SEG_AD_V10
else:
debug_seg_ad = DEBUG_SEG_AD_V9
comp_timestamp = get_long(O_TS, byteswapped)
if version == 11:
header1_size = get_long(O_HDR11, byteswapped)
else:
header1_size = get_long(O_HDR1, byteswapped)
text_segment_size = get_long(O_SEGT_SZ, byteswapped)
# timestamp is in localtime
#
char_date = datetime.datetime.fromtimestamp(comp_timestamp).strftime("%a %b %d %H:%M:%S %Y")
pointer = pointer + HDR_SZ + header1_size
# there is a bug here with what appears to be really old v9 dotr's that causes this to grok
try:
i_offset = get_long(I_SEG_AD, byteswapped) # usually works out to zero
except:
if not do_export and not do_compare:
print("Unable to get debugging segment offset in %s \n" % (file_to_open))
return
opointer = pointer
debug_offset = get_long(debug_seg_ad, byteswapped)
#if debug_offset == 0:
# if not (do_export and do_compare):
# print("No debug segment in %s \n" % (file_to_open))
# return
# p points to i segment
pointer = opointer + text_segment_size + i_offset
prog_crc = get_short(o_prog_crc, byteswapped) # main code r-code crc
pointer = opointer + text_segment_size + debug_offset
src_count = get_short(o_src_cnt, byteswapped) # number of include files
src_list = get_var_size(O_SRC_LST, sz_src_lst, byteswapped)
struct_addr = src_list
includes = [] # include list of tuples
for _ in range(0, src_count):
include_crc = get_short(struct_addr + o_debsrc_crc, byteswapped) # good!
full_path_ptr = pointer + struct_addr + o_debsrc_file
file_path = get_string_a(full_path_ptr, get_byte_a(full_path_ptr - 1))
file_name_ptr = pointer + struct_addr + o_debsrc_file + len(file_path)
file_name = get_string_terminated(file_name_ptr)
# now build the address of the next file reference
struct_addr = get_var_size(struct_addr + O_DEBSRC_NEXT, sz_debsrc_next, byteswapped)
includes.append((file_name, file_path, include_crc))
if do_sort: # reorder output list
includes = sorted(includes, key=sort_key) #Sort on the module name
if do_compare:
xname = os.path.basename(file_to_open)
xptr = 0 # 0 = left column 1 = right
if not primary_list:
xptr = 1
if not xname in comp_result:
comp_result[xname] = [0,0, {}] # Create entry for the output report
comp_result[xname][xptr] = prog_crc
# now get the detail for this one and put into the same dictionary entry
for inc in includes:
iname = inc[0]
if not iname in comp_result[xname][2]:
comp_result[xname][2][iname] = [0,0]
comp_result[xname][2][iname][xptr] = inc[2]
else:
if not do_export:
if not do_tab:
# old familiar printout
print("R-Code file: %s" % file_to_open)
print("Size: %s" % len(p))
print("Compiled on: %i (0x%lx) %s" % (comp_timestamp, comp_timestamp, char_date))
print("R-Code version: %s" % str(rcode_version))
print("R-Code CRC: %s" % str(prog_crc))
else:
# progress export format dotr information
x = ''
x += edq("%s" % (file_to_open)) # original filename to check
x += " %s" % (len(p)) # filesize
x += " %s" % (datetime.datetime.fromtimestamp(comp_timestamp).isoformat())
x += " %s" % str(rcode_version)
x += " %s" % str(prog_crc)
print x
if not do_headers_only:
for inc in includes:
if do_export:
# progress export format include references
x = ''
x += edq("%s" % (file_to_open)) # original filename to check
x += " " + edq("%s" % (inc[0])) # include basename
x += " " + edq("%s%s" % (inc[1],inc[0])) # reference path + name
x += " " + "%s" % (inc[2]) # include crc
print(x)
else:
if do_tab:
print("Source CRC:\t%s\t%s%s\t%s" % (inc[0], inc[1],inc[0], inc[2]))
else:
print("Source CRC: %s %s%s %s" % (inc[0], inc[1],inc[0], inc[2]))
if do_export and not do_headers_only:
print(".")
def read_dotr(filename):
try:
f = open(filename, "rb") # read binary
raw_data = f.read() # read as byte strings
f.close()
return array.array('i', map(ord,raw_data)) # convert to array int
except:
raise
def get_file_list(dir_or_file):
"""return the list of files to be processed """
try:
if os.path.isdir(dir_or_file):
temp_list = os.listdir(dir_or_file)
return [os.path.join(dir_or_file, i) for i in temp_list if i.endswith('.r')]
else: # single filename
return [dir_or_file]
except:
raise
def process_file(filename, read_type):
global p, errors_encountered
if os.path.isfile(filename):
try:
p = None # reset for each iteration
try: # read in the binary dotr file
p = read_dotr(filename)
except:
if read_type:
raise
else:
errors_encountered = True
except:
# if error on export mode silently fail
if read_type:
if not do_export:
print("ERROR: Unable to read %s \n" %(filename))
if len(file_list) == 1:
sys.exit(1)
errors_encountered = True
get_procrc(filename, read_type) # true indicate that this is the primary list
else:
errors_encountered = True
if __name__ == '__main__':
file_list = [] # file list to do
clist = [] # compare to list
comp_result = {} # compare results dictionary
parser = OptionParser(
usage="%prog [--version --sort --tab --dir [path] --export [--only] --compare [--message] --help ] rcode.r [rcode2.r ...]",
description=program_credits,
version="%prog 1.0",
formatter=IndentedHelpFormatterWithNL() )
if sys.version_info<(2,4,0):
print "You need python 2.4 or later to run this script\n"
parser.print_help()
sys.exit(1)
parser.add_option("-s", "--sort",
action="store_true", dest="do_sort",
help="Sort include files by module name")
parser.add_option("-t", "--tab",
action="store_true", dest="do_tab",
help="Tab delimit include filenames")
parser.add_option("-d", "--dir", type='string',
action="append", default=[], dest="scan_dir",
help="Directory Name to scan. Repeat as needed")
parser.add_option('-e', '--export', action='store_true',
dest='do_export',
help='Output in export format')
parser.add_option('-o', '--only', action='store_true',
dest='do_headers_only',
help='Only the header information is exported')
parser.add_option('-c', '--compare', type='string',
action="append", default=[], dest="cdir",
help='Compare to file/dir, repeat as needed')
parser.add_option('-u', '--message', type='string',
dest="user_message",
help='User message to put on compare report')
# args may also contain the filenames to process
options, args = parser.parse_args()
if options.do_sort != None:
do_sort = True
if options.do_tab != None:
do_tab = True
if options.do_export != None:
do_export = True # progress export format
if options.do_headers_only != None:
if not do_export:
print "ERROR: Export (-e) must be specified to use headers only\n"
parser.print_help()
sys.exit(1)
do_headers_only = True # headers only in progress export format
# testing
if testing_mode:
#do_export = True
#args = [ '/apps/workspace/procrc/src/dotrs/crctest9.r', '/apps/workspace/procrc/src/dotrs/crctest10.r', '/apps/workspace/procrc/src/dotrs/crctest11.r']
#options.scan_dir = [ '/apps/test/dir' ]
#options.user_message = 'first test'
pass
# read the dir names list to compare from
if len(options.scan_dir) != 0:
for xdir in options.scan_dir:
try:
file_list.extend(get_file_list(xdir))
except:
print "file/directory %s was not found" % (xdir)
if len(file_list) > 0: from_dir = options.scan_dir[0]
# read any filenames on the command line
if len(args) != 0:
for xfile in args:
file_list.append(xfile)
# read the directory/file names to compare to
if len(options.cdir) != 0:
do_compare = True
for xdir in options.cdir:
try: # build out the list to compare to
clist.extend(get_file_list(xdir))
except:
print "Compare file/directory %s was not found" % (xdir)
do_compare = False
if len(clist) > 0: to_dir = options.cdir[0]
# if doing compare turn off anything to do with exporting
if do_compare:
do_export = False
do_headers_only = False
if not do_export and not do_compare:
print(copyright)
# if nothing to do then print and exit
if len(file_list) == 0:
if do_export:
print(".")
print(".")
else:
print("Nothing to do!")
sys.exit(1)
# read and process the 'from' files
for filename in file_list:
process_file(filename, READ_FROM_FILES)
if len(args) > 1 or len(file_list) > 1:
if not do_export and not do_compare:
print(" ") # put a blank line between them
# read and process the compare to list
#
if len(clist) > 0:
# read in the compare to list
for filename in clist:
process_file(filename, READ_TO_FILES)
# now we have both sides
# a zero in the results list, indicates that the file was not found
files_checked = len(comp_result)
diffs_found = 0
# clean out the comparison structure of matching entries
# leaving only the differences to report
for xkey in comp_result.keys():
if comp_result[xkey][0] == comp_result[xkey][1]:
# eliminate all those that are the same program level crc
try:
del comp_result[xkey]
except KeyError:
continue
else:
include_dict = comp_result[xkey][2]
for ykey in include_dict:
if include_dict[ykey][0] == include_dict[ykey][1]:
include_dict[ykey] = [ 0, 0 ] # a hack
diffs_found += 1
# display the results
print(copyright)
# now report the results of the comparison
if options.user_message != None:
print "%-80s" % (options.user_message)
if from_dir != None:
print "Dir %-s" % (from_dir)
if options.cdir != None:
print "CDir %-s" % (to_dir)
if len(comp_result) != 0:
print "%-25s %-10s %-10s" % ('FileName', 'CRC Dir', 'CRC Cdir')
for key in sorted(comp_result):
print "%-25s %-10s %-10s" % (key, str(comp_result[key][0]), str(comp_result[key][1]))
includes = comp_result[key][2]
for include in sorted(includes):
# to work around dictionary size change issue
if includes[include][0] == 0 and includes[include][1] == 0: # skip if both are zero
continue
print " %-22s %-10s %-10s" % (include, str(includes[include][0]), str(includes[include][1]))
print " "
print "%d unique files checked, %d differences found." % (files_checked, diffs_found)
if do_export:
print(".")