Python Schedule Jar | Python Fiddle

from multiprocessing import Process
import getpass
import sys
import os, errno
import shutil
from time import gmtime, strftime
import re
from subprocess import Popen, PIPE, STDOUT
from pip._vendor.distlib._backport.tarfile import _FileInFile
from unittest.test import test_result
from test.badsyntax_future3 import result
from test.test_decimal import directory
from test.test_threading_local import target
import subprocess
from breeze.util import Sorting
from distutils import text_file
from org.apache.commons.math.optimization import direct
from pip._vendor.appdirs import props

def read_exec_func(AUD_FILE, audit_return_flag):
    output_file_count="zero"
    output_file_size="00000Bytes"
    try:
        with open(AUD_FILE,'r+') as infile:
            for l in infile:
                line = l.spilt('/')
                INP_FILE_NAME=line[0]
                Log_File_Name=INP_FILE_NAME.rsplit('/',1)[-1]
                exec_func(INP_FILE_NAME,Log_File_Name)
                if output_file_count=='Y':
                    output_file_count=count_file_recs(output_path+currDt+'/'+Log_File_Name+out_file_extn)
                if output_file_size=='Y':
                    output_file_size=file_size(output_path+currDt+'/'+Log_File_Name+out_file_extn)
                result = INP_FILE_NAME +'/'+strftime("%Y-%m-%d")+'|'+strftime("%H:%M:%S",gettime())+'|'+output_path+currDt+'|'+str(output_file_size)
                audit_write_onsuccess(result)
                if os.path.exists(ARCHV_DIR+'/'+currDt+'/'+Log_File_Name):
                    os.remove(ARCHV_DIR+'/'+currDt+'/'+Log_File_Name)
                shutil.move(INP_FILE_NAME, ARCHV_DIR+'/'+currDt)
    except IOError as ioex:
        print "Problem Reading "+AUD_FILE+" file"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)

def main_function(TEMP_AUDIT_FILE,audit_rerun_flag):
    try:
        if os.path.isfile(TEMP_AUDIT_FILE):
            rec_count=count_file_recs(TEMP_AUDIT_FILE)
            read_exec_func(TEMP_AUDIT_FILE, audit_return_flag)
            file_remove_dir(TEMP_AUDIT_FILE)
    except IOError as ioex:
        print "Problem executing Main function"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)
        
def audit_write_onsuccess(result):
    try:
        with open(AUDIT_FILE, 'a') as output_file:
            print result
            output_file.write(result)
            output_file.write("\n")
    except IOError as ioex:
        print "Problem writing "+AUD_FILE+" file"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)
    finally:
        output_file.close()
        
def read_temp_files(directory, batch_mode_enable,audit_rerun_flag):
    try:
        jobs = []
        for root, directories, files in os.walk(directory):
            for filename in files:
                filepath = os.path.join(root,filename)
                if batch_mode_enable=='Y':
                    if filepath.rsplit('_',1)[0]==TEMP_AUDIT_FILE:
                        p=Process(target=main_function,args=(filepath,audit_rerun_flag))
                        jobs.append(p)
                        p.start
                else:
                    main_function(TEMP_AUDIT_FILE, audit_rerun_flag)
                    
        for j in jobs:
            j.join()
            print ' %s.exitcode = %s ' %(j.name, j.exitcode)
    except IOError as ioex:
        print "Problem reading the read_temp_files "+TEMP_AUD_FILE+" file"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)                
                
                
def exec_func(INP_FILE_NAME, Log_File_Name):
    args= [jar_dir_path, INP_FILE_NAME, output_path+currDt+'/'+Log_File_Name+out_file_extn, out_file_delimiter]
    with open(LOG_DIR+'/'+currDt+'/'+Log_File_Name,'w') as fp :
        retcode= subprocess.Popen(['java','-jar'])+list(args),stdout=fp)
    streamdata = retcode.communicate()[0]
    #rc= retcode.returncode
    
    with open(LOG_DIR+'/'+currDt+'/'+Log_File_Name) as fp:
        output=f.read(1)
    print output
    
    if output='P':
        print " Dataset "+INP_FILE_NAME+" has been processed successfully. Check Output Direcotry "+output_path+currDt+'/'
    else:
        print " Dataset "+INP_FILE_NAME+" has been processed failed. Please check log file "+LOG_DIR+"/"+currDt+"/"+Log_File_Name+".log"
        sys.exit(1)

def count_file_recs(count_file):
    try:
        file_read=open(count_file,'r')
        rec_count=sum(1 for row in file_read)
        file_read.close()
    except IOError as ioex:
        print "Problem reading the "+count_file+" file"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)           
    return rec_count

def convert_bytes(num):
    for x in ['bytes','KB','MB','GB','TB']:
        if num < 1024.0:
            return "%3.1f %s" % (num,x)
        num /= 1024.0

def file_size(file_path):
    if os.path.isfile(file_path):
        file_info=os.stat(file_path)
        return convert_bytes(file_info.st_size)
    
def batch_split_files(num_files_batch,my_file):
    sorting=True
    hold_lines=[]
    with open(my_file,'r') as text_file:
        for row in text_file:
            hold_lines.append(row)
    outer_count = 1
    line_count = 0
    while sorting:
        count=0
        increament = (outer_count-1) * num_files_batch
        left = len(hold_lines) - increament
        file_name = my_file+"_"+str(outer_count)
        hold_new_lines =[]
        if left < num_files_batch:
            while count < left:
                hold_new_lines.append(hold_lines[line_count])
                count +=1
                line_count +=1
            sorting=False
        else:
            while count < num_files_batch:
                hold_new_lines.append(hold_lines[line_count])
                count +=1
                line_count +=1
        outer_count +=1
        with open(file_name, 'w') as next_file:
            for row in hold_new_lines:
                next_file.write(row)
                
def get_file_names(directory):
    try:
        temp_audit_write= open(TEMP_AUDIT_FILE,'w')
        for root, directories, files in os.walk(directory):
            for filename in files:
                filepath = os.path.join(root,filename)
                temp_audit_write.write(filepath+"/"+strftime("%Y-%m-%d")+'|'+strftime("%H:%M:%S",gettime())+'/'+'N')
                temp_audit_write.write("\n")
    except IOError as ioex:
        print "Problem reading the "+TEMP_AUDIT_FILE+" file"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)
    finally:
        temp_audit_write.close()

def delete_temp_files(directory):
    try:
        for root, directories, files in os.walk(directory):
            for filename in files:
                filepath = os.path.join(root,filename)
                if filepath.rsplit('_',1)[0]==TEMP_AUDIT_FILE.rsplit('_',1)[0]:
                    file_remove_dir(filepath)
    except IOError as ioex:
        print "Problem deleting the delete_temp_files "+TEMP_AUDIT_FILE+" file"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)
                   
def check_temp_exists(directory):
    rerun_check_flag='N'
    try:
        for root, directories, files in os.walk(directory):
            for filename in files:
                filepath = os.path.join(root,filename)
                if filepath.rsplit('_',1)[0]==TEMP_AUDIT_FILE:
                    rerun_check_flag='Y'
                    break
    except IOError as ioex:
        print "Problem reading the check_temp_exists "+TEMP_AUDIT_FILE+" file"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)
    return rerun_check_flag

def move_files_fromDir(inputDir, outputDir):
    try:
        for filename in os.listdir(inputDir):
            shutil.move(inputDir+filename, outputDir)
    except IOError as ioex:
        print "Problem moving files from input directory "+inputDir+" to output directory "+outputDir
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)

def create_Dir_func(dir_name):
    if os.path.exists(dir_name):
        try:
            os.makedirs(dir_name)
        except IOError as ioex:
        print "Problem creating the directory "+dir_name
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)    
                
def file_remove_dir(delete_file):
    try:
        if os.path.isfile(delete_file):
            os.remove(delete_file)
    
    except IOError as ioex:
        print "Problem deleting the file "+delete_file
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)

def load_properties(PRM_FILE):
    sep='='
    comment_char='#'
    try:
        props={}
        with open(PRM_FILE, "rt") as f:
            for line in f:
                l=line.strip()
                if l and not l.startswith(comment_char):
                    key_value = ls.split(sep)
                    key = key_value[0].strip()
                    value=sep.join(key_value[1:].strip().strip('"'))
                    props[key] = value 
        return props
    except IOError as ioex:
        print "Problem reading the properties file "+PRM_FILE
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1) 
    
 
######## Main ########

currDt=strftime("%Y-%m-%d")
currTime=strftime("%H:%M:%S",gmtime())
output_file_count=0
output_file_size="0Bytes"
rerun_flag='N'
PRM_FILE='/home/prm.txt'

#Reading PRM file properties
load_properties(PRM_FILE)
input_path=parms.get('input_directory_path')
output_path=parms.get('output_directory_path')
audit_file_name=parms.get('audit_file_name')
jar_dir_path=parms.get('jar_dir_path')
output_file_delimiter=parms.get('output_file_delimiter')
output_file_extn=parms.get('output_file_extn')
home_dir_path=parms.get('home_directory_path')
job_run_num=parms.get('job_run_number')
output_file_count_flag=parms.get('output_file_count_flag')
output_file_size_flag=parms.get('output_file_size_flag')
str_num_files_batch=parms.get('num_files_per_batch')
num_files_batch=int(str_num_files_batch)
batch_mode_enable=parms.get('enable_batch_mode')

#Setting up Directory Paths
LOG_DIR=home_dir_path+"Logs"
AUDIT_DIR=home_dir_path+"Audit_Logs"
PROCESS_DIR=home_dir_path+"process_dir"
ARCHV_DIR=home_dir_path+"archive_dir"

#Setting up File Paths
TEMP_AUDIT_FILE=AUDIT_DIR+'/'+"rundate="+currDt+'/'+"temp_"+aud_file_name+"_"+job_run_num
AUDIT_FILE=AUDIT_DIR+'/'+"rundate="+currDt+'/'+audit_file_name+"_"+currDt+"_"+strftime("%H%M%S",gmtime())+"_"+job_run_num

create_Dir_func(LOG_DIR+'/'+currDt)
create_Dir_func(AUDIT_DIR+'/'+"rundate="+currDt)
create_Dir_func(output_path+'/'+currDt)
create_Dir_func(PROCESS_DIR)

print "script started at "+currDt+"  "+currTime

rerun_flag=check_temp_exists(AUDIT_DIR+'/'+currDt)

print "batch_mode_enable : " +batch_mode_enable

print "rerun_flag : "+rerun_flag

if batch_mode_enable=='Y' and rerun_flag=='N':
    try:
        print "Batch Mode and no rerun "
        if num_files_batch!="" and files_batch!=0:
            move_files_fromDir(input_path, PROCESS_DIR)
            get_file_names(PROCESS_DIR)
            print "TEMP_AUDIT_FILE "+ TEMP_AUDIT_FILE
            batch_split_files(num_files_batch,TEMP_AUDIT_FILE)
            file_remove_dir(TEMP_AUDIT_FILE)
            read_temp_files(AUDIT_FILE, batch_mode_enable, rerun_flag)
        else:
            print "num_files_per_batch should not be blank as the parameters - batch_mode_enabled to 'Y' and rerun flag to 'N'"
            sys.exit(1)
    except IOError as ioex:
        print "Problem while processing batch mode in normal run"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)        
elif batch_mode_enable=='N' and rerun_flag=='N':
    print " Normal run with no batch mode "
    try:
        TEMP_AUDIT_FILE=TEMP_AUDIT_FILE+'_'+'0'
        move_files_fromDir(input_path, PROCESS_DIR)
        get_file_names(PROCESS_DIR)
        read_temp_files(AUDIT_FILE, batch_mode_enable, rerun_flag)
    except IOError as ioex:
        print "Problem while processing non batch mode and no rerun"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)            
    
elif batch_mode_enable=='Y' and rerun_flag=='Y':
    try:
        print " Batch and Restart"
        get_file_names(PROCESS_DIR)
        batch_split_files(num_files_batch,TEMP_AUDIT_FILE)
        read_temp_files(AUDIT_FILE, batch_mode_enable, rerun_flag)
        delete_temp_files(AUDIT_DIR+'/'+currDt)
    except IOError as ioex:
        print "Problem while processing batch mode and rerun"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)    
        
else:
    try:
        print" Batch Mode = 'N' and Rerun 'Y'"
        TEMP_AUDIT_FILE=TEMP_AUDIT_FILE+'_'+'0'
        get_file_names(PROCESS_DIR)
        read_temp_files(AUDIT_FILE, batch_mode_enable, rerun_flag)
        delete_temp_files(AUDIT_DIR+'/'+currDt)
    except IOError as ioex:
        print "Problem while processing no batch mode and rerun"
        print "ErrorNo: ", ioex.errno
        print(ioex)
        sys.exit(1)  
        
print " :::::::: Script completed successfully  at "+strftime("%Y-%m-%-d")+" "+strftime("%H:%M:%S",gmtime())
sys.exit(0)

Python Fiddle

Python Cloud IDE