From 6cb58e2a16421ce9740ecaf16f7322aac8a0aa03 Mon Sep 17 00:00:00 2001 From: Donata Mielaikaite Date: Fri, 7 Sep 2018 16:27:43 +0200 Subject: [PATCH 001/192] testing if I am able to push into repo --- test | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test diff --git a/test b/test new file mode 100644 index 0000000..e69de29 From 78435914a2778390090454cc63db00afcc92917c Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Wed, 3 Apr 2019 15:59:13 +0200 Subject: [PATCH 002/192] updated AAA files --- .gitignore | 4 + AAAOps/Federation/create-fedfiles.sh | 3 +- AAAOps/Federation/create_allow-list.sh | 2 +- AAAOps/Federation/create_fedmaps.py | 107 +++++++++++ AAAOps/Federation/exceptions.json | 52 ++++- AAAOps/Federation/hostIPv4.txt | 223 ++++++++++++++++++++++ AAAOps/Federation/transit-hostIPv4.txt | 17 ++ AAAOps/XfedKibana/XRDFED-kibana-probe.py, | 0 8 files changed, 399 insertions(+), 9 deletions(-) create mode 100755 AAAOps/Federation/create_fedmaps.py create mode 100644 AAAOps/Federation/hostIPv4.txt create mode 100644 AAAOps/Federation/transit-hostIPv4.txt create mode 100644 AAAOps/XfedKibana/XRDFED-kibana-probe.py, diff --git a/.gitignore b/.gitignore index ba76c66..82c2e3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ works/ +AAAOps/Federation/in/ +AAAOps/Federation/out/ +AAAOps/tools/timestamp.sh + diff --git a/AAAOps/Federation/create-fedfiles.sh b/AAAOps/Federation/create-fedfiles.sh index 94158b8..a0ab436 100755 --- a/AAAOps/Federation/create-fedfiles.sh +++ b/AAAOps/Federation/create-fedfiles.sh @@ -20,12 +20,13 @@ if [ ! -r $PROD_hosts ] || [ ! -r $TRANS_hosts ]; then exit 1 fi -python /opt/TransferTeam/AAAOps/Federation/create_fedmaps_DEV.py +python /opt/TransferTeam/AAAOps/Federation/create_fedmaps.py if [ ! -r $FED_json ]; then echo "We have a problem creating JSON file.\n" exit 1 fi + cp /opt/TransferTeam/AAAOps/Federation/out/* /var/www/html/aaa-fedinfo/ exit 0; diff --git a/AAAOps/Federation/create_allow-list.sh b/AAAOps/Federation/create_allow-list.sh index 49131a3..277e523 100755 --- a/AAAOps/Federation/create_allow-list.sh +++ b/AAAOps/Federation/create_allow-list.sh @@ -24,7 +24,7 @@ for j in "${redirectors[@]}";do cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u > $FEDINFO/in/prod_$j.txt cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u >> $FEDINFO/in/prod_$j.txt - cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_eu_$j.allow + cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_eu_$j.allow cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_us_$j.allow rm hostIPv4.txt hostIPv6.txt for f in $(cat $FEDINFO/in/prod_$j.txt);do diff --git a/AAAOps/Federation/create_fedmaps.py b/AAAOps/Federation/create_fedmaps.py new file mode 100755 index 0000000..dddc396 --- /dev/null +++ b/AAAOps/Federation/create_fedmaps.py @@ -0,0 +1,107 @@ +try: import xml.etree.ElementTree as ET +except ImportError: from elementtree import ElementTree as ET +try: import json +except ImportError: import simplejson as json +import urllib2, httplib, sys + +# global vars: prod, trans : we create these text files, see run.py +# cmsTopology : static text from dashboard team, +# sites : from BDII +# output : {"prod" : [...], "trans" : [...], "nowhere" : [...]} + +output = {"prod" : [], "trans" : [], "nowhere" : []} + +def getDataFromURL(url, header = {}): + request = urllib2.Request(url, headers=header) + urlObj = urllib2.urlopen(request) + data = urlObj.read() + return data + +def getSites(): + XML = getDataFromURL('http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii') + XML = ET.fromstring(XML) + sites = XML.findall('atp_site') + ret = {} + for site in sites: + groups = site.findall('group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + services = site.findall('service') + ret[siteName] = {} + ret[siteName]['hosts'] = [] + ret[siteName]['name'] = site.attrib['name'] + for service in services: + serviceName = service.attrib['hostname'] + ret[siteName]['hosts'].append(serviceName) + return ret + +def parseHN(data): + parsedHNs = [] + for line in data.split('\n'): + if not len(line): continue + if ':' in line: line = line[:line.find(':')] + parsedHNs.append(line) + return parsedHNs + +# try exception if we have a problem with URL. + +try : + sites = getSites() +except Exception as e : + err={} + err["error"] = str(e) + print json.dumps(err) + sys.exit(1) + +#import expection dictionary +with open('/opt/TransferTeam/AAAOps/Federation/exceptions.json') as f: exc = f.read() +exc = json.loads(exc) + +def exception(name): + ret = None + for i in exc.keys(): + if i == name : return exc[i]['VOname'] + return ret + + +def siteName2CMSSiteName(dom): + ret = None + for cmsSite in sites.keys(): + ret = exception(dom) + if ret : + ret = str(ret) + return ret + if sites[cmsSite]['hosts'][0].find(dom) != -1: + #print cmsSite + return cmsSite + return ret + +if __name__ == "__main__": + # get domains + domains = {} + with open('/opt/TransferTeam/AAAOps/Federation/in/prod_domain.txt') as f: domains['prod'] = parseHN(f.read()) + with open('/opt/TransferTeam/AAAOps/Federation/in/trans_domain.txt') as f: domains['trans'] = parseHN(f.read()) + + # find CMS site name of prod sites + for federation in ['prod', 'trans']: + for i in domains[federation]: + cmsSiteName = siteName2CMSSiteName(i) + if cmsSiteName and not cmsSiteName in output[federation]: + output[federation].append(cmsSiteName) + + # special case for nowhere sites: if a site is not placed in both + # federations, move it into "nowhere" array + for cmsSite in sites.keys(): + if not cmsSite in output['prod'] and not cmsSite in output['trans']: + output["nowhere"].append(cmsSite) + if cmsSite == "T1_FR_CCIN2P3" and cmsSite in output['prod'] : + output["prod"].append("T2_FR_CCIN2P3") + + with open('/opt/TransferTeam/AAAOps/Federation/out/federations.json', 'w') as f: + f.write(json.dumps(output, indent = 1)) +exit() diff --git a/AAAOps/Federation/exceptions.json b/AAAOps/Federation/exceptions.json index adfb02a..833b28d 100644 --- a/AAAOps/Federation/exceptions.json +++ b/AAAOps/Federation/exceptions.json @@ -1,5 +1,5 @@ { - "ccxrpli001.in2p3.fr" : { + "ccxrpli003.in2p3.fr" : { "VOname" : "T1_FR_CCIN2P3" }, @@ -11,7 +11,7 @@ "VOname" : "T2_FR_IPHC" }, - "polgrfs20.in2p3.fr " : { + "polgrid4.in2p3.fr" : { "VOname" : "T2_FR_GRIF_LLR" }, @@ -19,19 +19,19 @@ "VOname" : "T2_US_Nebraska" }, - "ds-05-01.cr.cnaf.infn.it" : { + "ds-t3-01.cr.cnaf.infn.it" : { "VOname" : "T3_IT_Bologna" }, - "ds-05-02.cr.cnaf.infn.it" : { + "ds-t3-02.cr.cnaf.infn.it" : { "VOname" : "T3_IT_Bologna" }, - "ds-202-06-33.cnaf" : { + "xrootd-cms-01.cnaf" : { "VOname" : "T1_IT_CNAF" }, - "ds-202-06-37.cnaf" : { + "xrootd-cms-01.cnaf" : { "VOname" : "T1_IT_CNAF" }, @@ -43,6 +43,10 @@ "VOname" : "T2_UK_SGrid_RALPP" }, + "ic.ac.uk" : { + "VOname" : "T2_UK_London_IC" + }, + "xrootd.knu.ac.kr" : { "VOname" : "T2_KR_KNU" }, @@ -65,6 +69,40 @@ "cmsxrd.ts.infn.it" : { "VOname" : "T3_IT_Trieste" - } + }, + "se11.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + + "se9.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + + "se1.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + + "se4.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + + "se8.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + "purdue.edu" : { + "VOname" : "T2_US_Purdue" + }, + "cscs.ch" : { + "VOname" : "T2_CH_CSCS" + }, + "csc.fi" : { + "VOname" : "T2_FI_HIP" + }, + "eoscms-srv-m1.cern.ch" : { + "VOname" : "T2_CH_CERN" + }, + "cmsxrootd-2.gridka.de" : { + "VOname" : "T1_DE_KIT" + } } diff --git a/AAAOps/Federation/hostIPv4.txt b/AAAOps/Federation/hostIPv4.txt new file mode 100644 index 0000000..e207bd8 --- /dev/null +++ b/AAAOps/Federation/hostIPv4.txt @@ -0,0 +1,223 @@ +ccxrpli003.in2p3.fr +cms-aaa-manager01.gridpp.rl.ac.uk +cms-aaa-proxy695.gridpp.rl.ac.uk +cms-aaa-proxy719.gridpp.rl.ac.uk +cms-se0.kipt.kharkov.ua +cms-t2-se01.sdfarm.kr +cms02.lcg.cscs.ch +cmsrm-xrootd01.roma1.infn.it +cmsrm-xrootd02.roma1.infn.it +cmsxrootd-1.gridka.de +cmsxrootd-2.gridka.de +cmsxrootd.ihep.ac.cn +dc2-grid-64.brunel.ac.uk +ds-202-04-05.cr.cnaf.infn.it +ds-202-04-07.cr.cnaf.infn.it +ds-202-11-05.cr.cnaf.infn.it +ds-202-11-07.cr.cnaf.infn.it +eoscms-srv-b2.cern.ch +eoscms-srv-m1.cern.ch +eymir.grid.metu.edu.tr +gaexroot01.ciemat.es +gfe02.grid.hep.ph.ic.ac.uk +grid-vo-cms.physik.rwth-aachen.de +grid02.physics.uoi.gr +grid143.kfki.hu +grse001.inr.troitsk.ru +hask.csc.fi +hephyse.oeaw.ac.at +heplnx228.pp.rl.ac.uk +heplnx229.pp.rl.ac.uk +ingrid-se03.cism.ucl.ac.be +ingrid-se04.cism.ucl.ac.be +ingrid-se05.cism.ucl.ac.be +ingrid-se06.cism.ucl.ac.be +lcgse01.phy.bris.ac.uk +lcgsexrd.jinr.ru +llrxrd-redir.in2p3.fr +maite.iihe.ac.be +node12.datagrid.cea.fr +nute.csc.fi +polgrid4.in2p3.fr +pool01.ifca.es +pool02.ifca.es +pool03.ifca.es +pool04.ifca.es +pool05.ifca.es +pool06.ifca.es +pool07.ifca.es +pool08.ifca.es +sbgse1.in2p3.fr +se-xrd01.jinr-t1.ru +se.cis.gov.pl +se.grid.icm.edu.pl +se01.grid.nchc.org.tw +se01.indiacms.res.in +se3.itep.ru +seadmin.ihep.ac.cn +ss-01.recas.ba.infn.it +ss-02.recas.ba.infn.it +ss-03.recas.ba.infn.it +stormgf1.pi.infn.it +stormgf2.pi.infn.it +t2-cms-xrootd01.desy.de +t2-cms-xrootd02.desy.de +t2-cms-xrootd03.desy.de +t2-xrdcms.lnl.infn.it +vobox0002.m45.ihep.su +xroot-data01.ncg.ingrid.pt +xroot-data02.ncg.ingrid.pt +xroot.ncg.ingrid.pt +xrootd-cms-01.cr.cnaf.infn.it +xrootd-cms-02.cr.cnaf.infn.it +xrootd-cms-uk.gridpp.rl.ac.uk +xrootd-es-cie.ciemat.es +xrootd-es-pic.pic.es +xrootd.ba.infn.it +xrootd01-cmst1.pic.es +xrootd01.grid.hep.ph.ic.ac.uk +xrootd02-cmst1.pic.es +xrootd02.grid.hep.ph.ic.ac.uk +xrootd1.hep.kbfi.ee +xrootd2.hep.kbfi.ee +xrootd3.hep.kbfi.ee +cms-b005.rcac.purdue.edu +cms-b006.rcac.purdue.edu +cms-b007.rcac.purdue.edu +cms-b008.rcac.purdue.edu +cms-b009.rcac.purdue.edu +cms-b012.rcac.purdue.edu +cms-c000.rcac.purdue.edu +cms-c001.rcac.purdue.edu +cms-c002.rcac.purdue.edu +cms-c003.rcac.purdue.edu +cms-c004.rcac.purdue.edu +cms-c005.rcac.purdue.edu +cms-c006.rcac.purdue.edu +cms-c007.rcac.purdue.edu +cms-c008.rcac.purdue.edu +cms-c009.rcac.purdue.edu +cms-d000.rcac.purdue.edu +cms-d001.rcac.purdue.edu +cms-f000.rcac.purdue.edu +cms-f001.rcac.purdue.edu +cms-f002.rcac.purdue.edu +cms-f003.rcac.purdue.edu +cms-g000.rcac.purdue.edu +cms-g001.rcac.purdue.edu +cms-g002.rcac.purdue.edu +cms-g003.rcac.purdue.edu +cms-g004.rcac.purdue.edu +cms-i000.rcac.purdue.edu +cms-i001.rcac.purdue.edu +cms-i002.rcac.purdue.edu +cms-i003.rcac.purdue.edu +cms-i004.rcac.purdue.edu +cms-i005.rcac.purdue.edu +cms-i006.rcac.purdue.edu +cms-j000.rcac.purdue.edu +cms-j001.rcac.purdue.edu +cms-j002.rcac.purdue.edu +cms-j003.rcac.purdue.edu +cms-k000.rcac.purdue.edu +cms-k001.rcac.purdue.edu +cms-k002.rcac.purdue.edu +cms-k003.rcac.purdue.edu +cms-k004.rcac.purdue.edu +cms-k005.rcac.purdue.edu +cms-k006.rcac.purdue.edu +cms-k007.rcac.purdue.edu +cms-k008.rcac.purdue.edu +cms-k009.rcac.purdue.edu +cms-l000.rcac.purdue.edu +cms-l001.rcac.purdue.edu +cms-l002.rcac.purdue.edu +cms-m000.rcac.purdue.edu +cms-m001.rcac.purdue.edu +cmsio10.rc.ufl.edu +cmsio3.rc.ufl.edu +cmsio4.rc.ufl.edu +cmsio5.rc.ufl.edu +cmsio6.rc.ufl.edu +cmsio7.rc.ufl.edu +cmsio8.rc.ufl.edu +cmsio9.rc.ufl.edu +cmsxrootd-site1.fnal.gov +cmsxrootd-site2.fnal.gov +cmsxrootd-site3.fnal.gov +g22n01.hep.wisc.edu +g22n02.hep.wisc.edu +g22n03.hep.wisc.edu +g22n04.hep.wisc.edu +g22n05.hep.wisc.edu +g22n06.hep.wisc.edu +g22n07.hep.wisc.edu +g22n08.hep.wisc.edu +g22n09.hep.wisc.edu +g22n10.hep.wisc.edu +g22n11.hep.wisc.edu +g22n12.hep.wisc.edu +g22n13.hep.wisc.edu +g22n15.hep.wisc.edu +g22n16.hep.wisc.edu +g22n17.hep.wisc.edu +gftp-1.t2.ucsd.edu +gftp-2.t2.ucsd.edu +gftp-3.t2.ucsd.edu +osg-se.sprace.org.br +pubxrootd.hep.wisc.edu +red-gridftp1.unl.edu +red-gridftp10.unl.edu +red-gridftp11.unl.edu +red-gridftp12.unl.edu +red-gridftp2.unl.edu +red-gridftp3.unl.edu +red-gridftp4.unl.edu +red-gridftp5.unl.edu +red-gridftp6.unl.edu +red-gridftp7.unl.edu +red-gridftp8.unl.edu +red-gridftp9.unl.edu +redirector.t2.ucsd.edu +s17n01.hep.wisc.edu +s17n02.hep.wisc.edu +s17n04.hep.wisc.edu +s17n05.hep.wisc.edu +s17n07.hep.wisc.edu +s17n08.hep.wisc.edu +s17n09.hep.wisc.edu +se8.accre.vanderbilt.edu +sea.accre.vanderbilt.edu +seb.accre.vanderbilt.edu +sec.accre.vanderbilt.edu +sed.accre.vanderbilt.edu +sef.accre.vanderbilt.edu +seg.accre.vanderbilt.edu +seh.accre.vanderbilt.edu +spstrg04.sprace.org.br +spstrg05.sprace.org.br +spstrg06.sprace.org.br +spstrg07.sprace.org.br +spstrg08.sprace.org.br +spstrg09.sprace.org.br +test-003.t2.ucsd.edu +transfer-1.ultralight.org +transfer-2.ultralight.org +transfer-4.ultralight.org +transfer-5.ultralight.org +transfer-6.ultralight.org +transfer-8.ultralight.org +xrootd-local.unl.edu +xrootd.accre.vanderbilt.edu +xrootd.cmsaf.mit.edu +xrootd.rcac.purdue.edu +xrootd10.cmsaf.mit.edu +xrootd11.cmsaf.mit.edu +xrootd12.cmsaf.mit.edu +xrootd13.cmsaf.mit.edu +xrootd14.cmsaf.mit.edu +xrootd15.cmsaf.mit.edu +xrootd16.cmsaf.mit.edu +xrootd17.cmsaf.mit.edu +xrootd18.cmsaf.mit.edu +xrootd9.cmsaf.mit.edu diff --git a/AAAOps/Federation/transit-hostIPv4.txt b/AAAOps/Federation/transit-hostIPv4.txt new file mode 100644 index 0000000..78913c7 --- /dev/null +++ b/AAAOps/Federation/transit-hostIPv4.txt @@ -0,0 +1,17 @@ +bonner03.rice.edu +bonner04.rice.edu +cms-se.hep.uprm.edu +cmsxrd.ts.infn.it +ds-t3-01.cr.cnaf.infn.it +ds-t3-02.cr.cnaf.infn.it +hepcms-0.umd.edu +hepxrd01.colorado.edu +kodiak-se.baylor.edu +lyogrid06.in2p3.fr +ntugrid4.phys.ntu.edu.tw +ntugrid6.phys.ntu.edu.tw +pcncp22.ncp.edu.pk +ruhex-osgce.rutgers.edu +storm.mib.infn.it +t3se01.psi.ch +xrootd.knu.ac.kr diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py, b/AAAOps/XfedKibana/XRDFED-kibana-probe.py, new file mode 100644 index 0000000..e69de29 From f907ee884f282a28b7dbda77f9e096abb19af1a2 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Thu, 25 Apr 2019 14:09:15 +0200 Subject: [PATCH 003/192] implemented fix for parsing --- .gitignore | 2 +- AAAOps/Federation/.create_fedmaps_DEV.py.swo | Bin 0 -> 12288 bytes AAAOps/Federation/create_allow-list.sh | 13 ++-- AAAOps/Federation/hostIPv4.txt | 64 +++++++++---------- AAAOps/tools/timestamp.sh | 1 + 5 files changed, 41 insertions(+), 39 deletions(-) create mode 100644 AAAOps/Federation/.create_fedmaps_DEV.py.swo mode change 100644 => 100755 AAAOps/tools/timestamp.sh diff --git a/.gitignore b/.gitignore index 82c2e3d..fe2c38f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ works/ AAAOps/Federation/in/ AAAOps/Federation/out/ AAAOps/tools/timestamp.sh - +AAAOps/Federation/hostIPv4.txt diff --git a/AAAOps/Federation/.create_fedmaps_DEV.py.swo b/AAAOps/Federation/.create_fedmaps_DEV.py.swo new file mode 100644 index 0000000000000000000000000000000000000000..9be830c2060aadccf57e2db26842a9081192b8f3 GIT binary patch literal 12288 zcmeHNO^h5z6)uP0jcq`Q#3c$U&mwiNwC>qmBUv745;op7UddjAJ+^|zlTOcc%}(1h zU7hNh{gJgYLIMey0|!6?!_5+0FcLTH9Jp{wjzmhhAz?0v6PH**BHyd3p6OZJ80VBo z-O@MRUGLRA ziBmyOs5Iyeo}^!wnXb1~l;(Y*S|S?NS65drWO@CJh=q!DBGYA`; zl_1-DlEJFIVxVH+85lTv`NYcd5_Oob9pf*)aQPYfR;m>P6$2Fm6$2Fm6$2Fm6$2Fm z6$77k26T3a-NYcCa|5~N?uQQC-D`EL7^oPi7^oPi7^oPi7^oPi7^oPi7^oPi7^oQd z6fzLSjQt96ze)c9fdBt5{{Dae%Z&XJ_z}1#h4}2F`1$^M)VaEOp{2urj@IBx%&;Y&){Ox(h{si0t zTEIH61}p<#1O79|*ayHJ;0HhkNZ^~mtH3Fs20Rb^_lt-Xcn7!-+yj0Md<$3xmVlRl zBf!7E06pMs;688jQtMyAd36eY@7v3WGcDw~1 z<-FfuY>v!Yf9+bPWL)b6Q3emOUu&&|XD)`PPj76leEURgfz2g-j`J*+5G`*ud9Afv zmh&i$C*mY6jCy3$>+l96PsLSuOy$l%C^2>F?AzgzHhiX1{}03D_6ZuAPSkiE>$)#h z*-DO4^m)P zj5jFu$UO5TH!)_zs3&5WBdQ z$>MAxGKKLKyig}=yyOnkkA_3CmDA@hQTFNCPo_DdaIB$((KB5{M?xcw9K?$KkGCS>lcIi{%c)PdVxH*JF$(#OeMVf*^DUu^{tOc%42h#KG|My$ zDFI7$;=@6(hc|{8XG-Z4!qd51`u)v;iQclde zYV8U&F$Uws++*{}D3eN~86}8_sj8Mu-@X)T| zN2*!}J}7&rF=PXcO-wNx6JIZqMtg^F1>MXZ^uh$B%79|ZwAzD^&2xH~jECygj^Kj` zRW+ie+Z~FL-Ow&yJQvL5%q+F|>}GDFphzYy#pX0jQWqqNVcTnXR6?k|VSW)5y|f$j zYo{`F7HpPXCn}gj?54(AgZE^rlXNT?iD^PoyQyK3j)FWJf==&h>bVVL5)@puJ7!iE zMIuHAniFKGG!f~wxX|1t_BJtnrpvAvUoO;k(lbJvi=taeK*_RdL8P@xx_0rFCNp)S zaX2*^ihInb+rWW9Y}ekqob5u3O@TORKV5f?|AtwcsDt}F4@SmYYVlIu_;*Ep%%jgtfsP|2Rwh}o4vi-jRJP7ic% z^069T=w5R>Eu%Ku;$*|5yAq(qd}c)IBCnb6hg$JA`9>t^K2^Y(%Na3kjk2X62xg7BhO#|s+CON~ zrrkPeaLOuwZEa)Ypc<0Vl9?RJEfi+dcbq5haV>VW?P5|(2}D~oT14-yq3q(bb{k(j z4fB $BASE/tmp_euRED_$j xrdmapc --list all "$j" | grep -E 'cmsxrootd2.fnal.gov|xrootd.unl.edu' | awk '{print $3}' | cut -d ':' -f1 > $BASE/tmp_usRED_$j for i in $(cat $BASE/tmp_euRED_$j);do - xrdmapc --list all $i:1094 > $BASE/tmp_$i + xrdmapc --list all $i:1094 > $BASE/tmp_$i cat $BASE/tmp_$i | awk '{if($2=="Man") print $3; else print $2}' | tail -n +2 >> $BASE/tmp_total_eu_$j done @@ -23,9 +23,9 @@ for j in "${redirectors[@]}";do cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u > $FEDINFO/in/prod_$j.txt - cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u >> $FEDINFO/in/prod_$j.txt - cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_eu_$j.allow - cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_us_$j.allow + cat $BASE/tmp_total_us_$j | cut -d : -f1 | grep -v "\[" | sort -u >> $FEDINFO/in/prod_$j.txt + cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_eu_$j.allow + cat $BASE/tmp_total_us_$j | cut -d : -f1 | grep -v "\[" | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_us_$j.allow rm hostIPv4.txt hostIPv6.txt for f in $(cat $FEDINFO/in/prod_$j.txt);do if [ "$f" != "${1#*[0-9].[0-9]}" ]; then @@ -81,12 +81,13 @@ echo "* redirect cms-xrd-transit.cern.ch+:1213" >> $FEDINFO/out/list_eu.allow #cat $FEDINFO/in/prod.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "it" || $(NF-1) == "cern" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt #Quick fix for "[" character in prod.txt -cat $FEDINFO/in/prod.txt | awk '{ if ($1 == "[") print "Unknown.Host"; else print $1;}' > $FEDINFO/in/tmp +cat $FEDINFO/in/prod.txt | awk '{ if ($1 == "[2804") print "Unknown.Host"; else print $1;}' > $FEDINFO/in/tmp +cat $FEDINFO/in/prod.txt > $FEDINFO/in/tmp cp $FEDINFO/in/tmp $FEDINFO/in/prod.txt rm $FEDINFO/in/tmp -cat $FEDINFO/in/prod.txt | sort -u | awk -F. '{if ($NF == "uk" && $(NF-2) != "rl" || $NF == "fr" || $(NF-1) == "cern" || $(NF-1) == "fnal" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-2) == "cnaf") print $(NF-4)"."$(NF-2); else if ( $NF == "it" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-2) == "rl" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "kr") print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "be" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt +cat $FEDINFO/in/prod.txt | sort -u | awk -F. '{if ($NF == "uk" && $(NF-2) != "rl" || $NF == "fr" || $(NF-1) == "cern" || $(NF-1) == "fnal" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "it" && $(NF-2) == "cnaf") print $(NF-4)"."$(NF-2); else if ( $NF == "it" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "uk" && $(NF-2) == "rl" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "kr") print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "be" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt cat $FEDINFO/in/trans.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "kr" || $NF == "it" || $NF == "ch" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ($NF == "it" && $(NF-2) == "cnaf" ) print $(NF-4)"."$(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/trans_domain.txt diff --git a/AAAOps/Federation/hostIPv4.txt b/AAAOps/Federation/hostIPv4.txt index e207bd8..ea46f16 100644 --- a/AAAOps/Federation/hostIPv4.txt +++ b/AAAOps/Federation/hostIPv4.txt @@ -1,12 +1,11 @@ ccxrpli003.in2p3.fr +cms02.lcg.cscs.ch cms-aaa-manager01.gridpp.rl.ac.uk cms-aaa-proxy695.gridpp.rl.ac.uk cms-aaa-proxy719.gridpp.rl.ac.uk +cmsrm-xrootd02.roma1.infn.it cms-se0.kipt.kharkov.ua cms-t2-se01.sdfarm.kr -cms02.lcg.cscs.ch -cmsrm-xrootd01.roma1.infn.it -cmsrm-xrootd02.roma1.infn.it cmsxrootd-1.gridka.de cmsxrootd-2.gridka.de cmsxrootd.ihep.ac.cn @@ -20,9 +19,8 @@ eoscms-srv-m1.cern.ch eymir.grid.metu.edu.tr gaexroot01.ciemat.es gfe02.grid.hep.ph.ic.ac.uk -grid-vo-cms.physik.rwth-aachen.de -grid02.physics.uoi.gr grid143.kfki.hu +grid-vo-cms.physik.rwth-aachen.de grse001.inr.troitsk.ru hask.csc.fi hephyse.oeaw.ac.at @@ -48,32 +46,24 @@ pool06.ifca.es pool07.ifca.es pool08.ifca.es sbgse1.in2p3.fr -se-xrd01.jinr-t1.ru -se.cis.gov.pl -se.grid.icm.edu.pl se01.grid.nchc.org.tw se01.indiacms.res.in se3.itep.ru seadmin.ihep.ac.cn +se.cis.gov.pl +se.grid.icm.edu.pl +se-xrd01.jinr-t1.ru ss-01.recas.ba.infn.it ss-02.recas.ba.infn.it ss-03.recas.ba.infn.it stormgf1.pi.infn.it stormgf2.pi.infn.it +stormgf3.pi.infn.it t2-cms-xrootd01.desy.de t2-cms-xrootd02.desy.de t2-cms-xrootd03.desy.de t2-xrdcms.lnl.infn.it vobox0002.m45.ihep.su -xroot-data01.ncg.ingrid.pt -xroot-data02.ncg.ingrid.pt -xroot.ncg.ingrid.pt -xrootd-cms-01.cr.cnaf.infn.it -xrootd-cms-02.cr.cnaf.infn.it -xrootd-cms-uk.gridpp.rl.ac.uk -xrootd-es-cie.ciemat.es -xrootd-es-pic.pic.es -xrootd.ba.infn.it xrootd01-cmst1.pic.es xrootd01.grid.hep.ph.ic.ac.uk xrootd02-cmst1.pic.es @@ -81,6 +71,15 @@ xrootd02.grid.hep.ph.ic.ac.uk xrootd1.hep.kbfi.ee xrootd2.hep.kbfi.ee xrootd3.hep.kbfi.ee +xroot-data01.ncg.ingrid.pt +xroot-data02.ncg.ingrid.pt +xrootd.ba.infn.it +xrootd-cms-01.cr.cnaf.infn.it +xrootd-cms-02.cr.cnaf.infn.it +xrootd-cms-uk.gridpp.rl.ac.uk +xrootd-es-cie.ciemat.es +xrootd-es-pic.pic.es +xroot.ncg.ingrid.pt cms-b005.rcac.purdue.edu cms-b006.rcac.purdue.edu cms-b007.rcac.purdue.edu @@ -115,6 +114,14 @@ cms-i003.rcac.purdue.edu cms-i004.rcac.purdue.edu cms-i005.rcac.purdue.edu cms-i006.rcac.purdue.edu +cmsio10.rc.ufl.edu +cmsio3.rc.ufl.edu +cmsio4.rc.ufl.edu +cmsio5.rc.ufl.edu +cmsio6.rc.ufl.edu +cmsio7.rc.ufl.edu +cmsio8.rc.ufl.edu +cmsio9.rc.ufl.edu cms-j000.rcac.purdue.edu cms-j001.rcac.purdue.edu cms-j002.rcac.purdue.edu @@ -134,14 +141,6 @@ cms-l001.rcac.purdue.edu cms-l002.rcac.purdue.edu cms-m000.rcac.purdue.edu cms-m001.rcac.purdue.edu -cmsio10.rc.ufl.edu -cmsio3.rc.ufl.edu -cmsio4.rc.ufl.edu -cmsio5.rc.ufl.edu -cmsio6.rc.ufl.edu -cmsio7.rc.ufl.edu -cmsio8.rc.ufl.edu -cmsio9.rc.ufl.edu cmsxrootd-site1.fnal.gov cmsxrootd-site2.fnal.gov cmsxrootd-site3.fnal.gov @@ -162,14 +161,12 @@ g22n15.hep.wisc.edu g22n16.hep.wisc.edu g22n17.hep.wisc.edu gftp-1.t2.ucsd.edu -gftp-2.t2.ucsd.edu -gftp-3.t2.ucsd.edu osg-se.sprace.org.br pubxrootd.hep.wisc.edu -red-gridftp1.unl.edu red-gridftp10.unl.edu red-gridftp11.unl.edu red-gridftp12.unl.edu +red-gridftp1.unl.edu red-gridftp2.unl.edu red-gridftp3.unl.edu red-gridftp4.unl.edu @@ -186,6 +183,8 @@ s17n05.hep.wisc.edu s17n07.hep.wisc.edu s17n08.hep.wisc.edu s17n09.hep.wisc.edu +s17n10.hep.wisc.edu +se1.accre.vanderbilt.edu se8.accre.vanderbilt.edu sea.accre.vanderbilt.edu seb.accre.vanderbilt.edu @@ -206,11 +205,8 @@ transfer-2.ultralight.org transfer-4.ultralight.org transfer-5.ultralight.org transfer-6.ultralight.org +transfer-7.ultralight.org transfer-8.ultralight.org -xrootd-local.unl.edu -xrootd.accre.vanderbilt.edu -xrootd.cmsaf.mit.edu -xrootd.rcac.purdue.edu xrootd10.cmsaf.mit.edu xrootd11.cmsaf.mit.edu xrootd12.cmsaf.mit.edu @@ -221,3 +217,7 @@ xrootd16.cmsaf.mit.edu xrootd17.cmsaf.mit.edu xrootd18.cmsaf.mit.edu xrootd9.cmsaf.mit.edu +xrootd.accre.vanderbilt.edu +xrootd.cmsaf.mit.edu +xrootd-local.unl.edu +xrootd.rcac.purdue.edu diff --git a/AAAOps/tools/timestamp.sh b/AAAOps/tools/timestamp.sh old mode 100644 new mode 100755 index ab57df7..c3c0590 --- a/AAAOps/tools/timestamp.sh +++ b/AAAOps/tools/timestamp.sh @@ -1,4 +1,5 @@ #!/bin/bash + while read x; do echo -n `date +%d/%m/%Y\ %H:%M:%S`; echo -n " "; From fc1a926887cc46346b70f311a49efe0d71175d20 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Thu, 25 Apr 2019 14:09:15 +0200 Subject: [PATCH 004/192] implemented fix for parsing From ada1572c637561a475612e1f179c450808b00372 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 30 Apr 2019 10:57:08 +0200 Subject: [PATCH 005/192] removed line added by mistake --- AAAOps/Federation/create_allow-list.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/AAAOps/Federation/create_allow-list.sh b/AAAOps/Federation/create_allow-list.sh index c16703f..ac15c0f 100755 --- a/AAAOps/Federation/create_allow-list.sh +++ b/AAAOps/Federation/create_allow-list.sh @@ -82,7 +82,6 @@ echo "* redirect cms-xrd-transit.cern.ch+:1213" >> $FEDINFO/out/list_eu.allow #Quick fix for "[" character in prod.txt cat $FEDINFO/in/prod.txt | awk '{ if ($1 == "[2804") print "Unknown.Host"; else print $1;}' > $FEDINFO/in/tmp -cat $FEDINFO/in/prod.txt > $FEDINFO/in/tmp cp $FEDINFO/in/tmp $FEDINFO/in/prod.txt rm $FEDINFO/in/tmp From 4be2ff47fea5a9a2546332c7d56ef84e24525b0e Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 30 Apr 2019 12:51:23 +0200 Subject: [PATCH 006/192] added regex for parsing --- AAAOps/Federation/create_allow-list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/Federation/create_allow-list.sh b/AAAOps/Federation/create_allow-list.sh index ac15c0f..63462b6 100755 --- a/AAAOps/Federation/create_allow-list.sh +++ b/AAAOps/Federation/create_allow-list.sh @@ -81,7 +81,7 @@ echo "* redirect cms-xrd-transit.cern.ch+:1213" >> $FEDINFO/out/list_eu.allow #cat $FEDINFO/in/prod.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "it" || $(NF-1) == "cern" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt #Quick fix for "[" character in prod.txt -cat $FEDINFO/in/prod.txt | awk '{ if ($1 == "[2804") print "Unknown.Host"; else print $1;}' > $FEDINFO/in/tmp +cat $FEDINFO/in/prod.txt | awk '{ if ($1 ~ /\[+/ ) print "Unknown.Host"; else print $1;}' > $FEDINFO/in/tmp cp $FEDINFO/in/tmp $FEDINFO/in/prod.txt rm $FEDINFO/in/tmp From daf1845554b940badb071a6fef0ecda53489139d Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 25 Jun 2019 11:26:42 +0200 Subject: [PATCH 007/192] adding left changes --- .gitignore | 3 +++ AAAOps/Federation/create_allow-list.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fe2c38f..a87d7df 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ works/ AAAOps/Federation/in/ AAAOps/Federation/out/ AAAOps/tools/timestamp.sh +../Federation/hostIPv4.txt +../Federation/transit-hostIPv4.txt AAAOps/Federation/hostIPv4.txt +AAAOps/Federation/transit-hostIPv4.txt diff --git a/AAAOps/Federation/create_allow-list.sh b/AAAOps/Federation/create_allow-list.sh index 63462b6..bff1d1d 100755 --- a/AAAOps/Federation/create_allow-list.sh +++ b/AAAOps/Federation/create_allow-list.sh @@ -38,7 +38,7 @@ for j in "${redirectors[@]}";do done else xrdmapc --list all "$j" | tail -n +2 | awk '{if($2=="Man") print $3; else print $2}' > $BASE/tmp_total - cat $BASE/tmp_total | cut -d : -f1 | sort -u > $FEDINFO/in/trans.txt + cat $BASE/tmp_total | cut -d : -f1 | grep -v "\[" | sort -u > $FEDINFO/in/trans.txt rm transit-hostIPv4.txt transit-hostIPv6.txt for f in $(cat $FEDINFO/in/trans.txt);do From 0cece03a0742aa5102a26349f2f039b5924cfd0e Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 25 Jun 2019 11:26:42 +0200 Subject: [PATCH 008/192] removing not needed files --- .gitignore | 1 - AAAOps/Federation/create_fedmaps_DEV.py | 105 ---------------------- AAAOps/XfedKibana/XRDFED-kibana-probe.py, | 0 3 files changed, 106 deletions(-) delete mode 100755 AAAOps/Federation/create_fedmaps_DEV.py delete mode 100644 AAAOps/XfedKibana/XRDFED-kibana-probe.py, diff --git a/.gitignore b/.gitignore index a87d7df..38ba786 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -works/ AAAOps/Federation/in/ AAAOps/Federation/out/ AAAOps/tools/timestamp.sh diff --git a/AAAOps/Federation/create_fedmaps_DEV.py b/AAAOps/Federation/create_fedmaps_DEV.py deleted file mode 100755 index 798d161..0000000 --- a/AAAOps/Federation/create_fedmaps_DEV.py +++ /dev/null @@ -1,105 +0,0 @@ -try: import xml.etree.ElementTree as ET -except ImportError: from elementtree import ElementTree as ET -try: import json -except ImportError: import simplejson as json -import urllib2, httplib, sys - -# global vars: prod, trans : we create these text files, see run.py -# cmsTopology : static text from dashboard team, -# sites : from BDII -# output : {"prod" : [...], "trans" : [...], "nowhere" : [...]} - -output = {"prod" : [], "trans" : [], "nowhere" : []} - -def getDataFromURL(url, header = {}): - request = urllib2.Request(url, headers=header) - urlObj = urllib2.urlopen(request) - data = urlObj.read() - return data - -def getSites(): - XML = getDataFromURL('http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii') - XML = ET.fromstring(XML) - sites = XML.findall('atp_site') - ret = {} - for site in sites: - groups = site.findall('group') - siteName = None - for i in groups: - if i.attrib['type'] == 'CMS_Site': - siteName = groups[1].attrib['name'] - break - if not siteName: - continue - services = site.findall('service') - ret[siteName] = {} - ret[siteName]['hosts'] = [] - ret[siteName]['name'] = site.attrib['name'] - for service in services: - serviceName = service.attrib['hostname'] - ret[siteName]['hosts'].append(serviceName) - return ret - -def parseHN(data): - parsedHNs = [] - for line in data.split('\n'): - if not len(line): continue - if ':' in line: line = line[:line.find(':')] - parsedHNs.append(line) - return parsedHNs - -# try exception if we have a problem with URL. - -try : - sites = getSites() -except Exception as e : - err={} - err["error"] = str(e) - print json.dumps(err) - sys.exit(1) - -#import expection dictionary -with open('/opt/TransferTeam/AAAOps/Federation/exceptions.json') as f: exc = f.read() -exc = json.loads(exc) - -def exception(name): - ret = None - for i in exc.keys(): - if i == name : return exc[i]['VOname'] - return ret - - -def siteName2CMSSiteName(dom): - ret = None - for cmsSite in sites.keys(): - ret = exception(dom) - if ret : - ret = str(ret) - return ret - if sites[cmsSite]['hosts'][0].find(dom) != -1: return cmsSite - return ret - -if __name__ == "__main__": - # get domains - domains = {} - with open('/opt/TransferTeam/AAAOps/Federation/in/prod_domain.txt') as f: domains['prod'] = parseHN(f.read()) - with open('/opt/TransferTeam/AAAOps/Federation/in/trans_domain.txt') as f: domains['trans'] = parseHN(f.read()) - - # find CMS site name of prod sites - for federation in ['prod', 'trans']: - for i in domains[federation]: - cmsSiteName = siteName2CMSSiteName(i) - #print cmsSiteName - if cmsSiteName and not cmsSiteName in output[federation]: - output[federation].append(cmsSiteName) - - # special case for nowhere sites: if a site is not placed in both - # federations, move it into "nowhere" array - for cmsSite in sites.keys(): - if not cmsSite in output['prod'] and not cmsSite in output['trans']: - output["nowhere"].append(cmsSite) - if cmsSite == "T1_FR_CCIN2P3" and cmsSite in output['prod'] : - output["prod"].append("T2_FR_CCIN2P3") - - with open('/opt/TransferTeam/AAAOps/Federation/out/federations.json', 'w') as f: - f.write(json.dumps(output, indent = 1)) diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py, b/AAAOps/XfedKibana/XRDFED-kibana-probe.py, deleted file mode 100644 index e69de29..0000000 From 982a05e2f38d28840cfc5dddb715be13dec8d3cf Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 25 Jun 2019 11:38:41 +0200 Subject: [PATCH 009/192] removed not needed files --- AAAOps/Federation/create_fedmaps_DEV.py | 105 ---------------------- AAAOps/XfedKibana/XRDFED-kibana-probe.py, | 0 2 files changed, 105 deletions(-) delete mode 100755 AAAOps/Federation/create_fedmaps_DEV.py delete mode 100644 AAAOps/XfedKibana/XRDFED-kibana-probe.py, diff --git a/AAAOps/Federation/create_fedmaps_DEV.py b/AAAOps/Federation/create_fedmaps_DEV.py deleted file mode 100755 index 798d161..0000000 --- a/AAAOps/Federation/create_fedmaps_DEV.py +++ /dev/null @@ -1,105 +0,0 @@ -try: import xml.etree.ElementTree as ET -except ImportError: from elementtree import ElementTree as ET -try: import json -except ImportError: import simplejson as json -import urllib2, httplib, sys - -# global vars: prod, trans : we create these text files, see run.py -# cmsTopology : static text from dashboard team, -# sites : from BDII -# output : {"prod" : [...], "trans" : [...], "nowhere" : [...]} - -output = {"prod" : [], "trans" : [], "nowhere" : []} - -def getDataFromURL(url, header = {}): - request = urllib2.Request(url, headers=header) - urlObj = urllib2.urlopen(request) - data = urlObj.read() - return data - -def getSites(): - XML = getDataFromURL('http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii') - XML = ET.fromstring(XML) - sites = XML.findall('atp_site') - ret = {} - for site in sites: - groups = site.findall('group') - siteName = None - for i in groups: - if i.attrib['type'] == 'CMS_Site': - siteName = groups[1].attrib['name'] - break - if not siteName: - continue - services = site.findall('service') - ret[siteName] = {} - ret[siteName]['hosts'] = [] - ret[siteName]['name'] = site.attrib['name'] - for service in services: - serviceName = service.attrib['hostname'] - ret[siteName]['hosts'].append(serviceName) - return ret - -def parseHN(data): - parsedHNs = [] - for line in data.split('\n'): - if not len(line): continue - if ':' in line: line = line[:line.find(':')] - parsedHNs.append(line) - return parsedHNs - -# try exception if we have a problem with URL. - -try : - sites = getSites() -except Exception as e : - err={} - err["error"] = str(e) - print json.dumps(err) - sys.exit(1) - -#import expection dictionary -with open('/opt/TransferTeam/AAAOps/Federation/exceptions.json') as f: exc = f.read() -exc = json.loads(exc) - -def exception(name): - ret = None - for i in exc.keys(): - if i == name : return exc[i]['VOname'] - return ret - - -def siteName2CMSSiteName(dom): - ret = None - for cmsSite in sites.keys(): - ret = exception(dom) - if ret : - ret = str(ret) - return ret - if sites[cmsSite]['hosts'][0].find(dom) != -1: return cmsSite - return ret - -if __name__ == "__main__": - # get domains - domains = {} - with open('/opt/TransferTeam/AAAOps/Federation/in/prod_domain.txt') as f: domains['prod'] = parseHN(f.read()) - with open('/opt/TransferTeam/AAAOps/Federation/in/trans_domain.txt') as f: domains['trans'] = parseHN(f.read()) - - # find CMS site name of prod sites - for federation in ['prod', 'trans']: - for i in domains[federation]: - cmsSiteName = siteName2CMSSiteName(i) - #print cmsSiteName - if cmsSiteName and not cmsSiteName in output[federation]: - output[federation].append(cmsSiteName) - - # special case for nowhere sites: if a site is not placed in both - # federations, move it into "nowhere" array - for cmsSite in sites.keys(): - if not cmsSite in output['prod'] and not cmsSite in output['trans']: - output["nowhere"].append(cmsSite) - if cmsSite == "T1_FR_CCIN2P3" and cmsSite in output['prod'] : - output["prod"].append("T2_FR_CCIN2P3") - - with open('/opt/TransferTeam/AAAOps/Federation/out/federations.json', 'w') as f: - f.write(json.dumps(output, indent = 1)) diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py, b/AAAOps/XfedKibana/XRDFED-kibana-probe.py, deleted file mode 100644 index e69de29..0000000 From a4a7120d6d090e8fc3026b306a4150dccfe3a672 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 25 Jun 2019 11:44:14 +0200 Subject: [PATCH 010/192] permanently ignore changes of host.txt files --- AAAOps/Federation/hostIPv4.txt | 223 ------------------------- AAAOps/Federation/transit-hostIPv4.txt | 17 -- 2 files changed, 240 deletions(-) delete mode 100644 AAAOps/Federation/hostIPv4.txt delete mode 100644 AAAOps/Federation/transit-hostIPv4.txt diff --git a/AAAOps/Federation/hostIPv4.txt b/AAAOps/Federation/hostIPv4.txt deleted file mode 100644 index ea46f16..0000000 --- a/AAAOps/Federation/hostIPv4.txt +++ /dev/null @@ -1,223 +0,0 @@ -ccxrpli003.in2p3.fr -cms02.lcg.cscs.ch -cms-aaa-manager01.gridpp.rl.ac.uk -cms-aaa-proxy695.gridpp.rl.ac.uk -cms-aaa-proxy719.gridpp.rl.ac.uk -cmsrm-xrootd02.roma1.infn.it -cms-se0.kipt.kharkov.ua -cms-t2-se01.sdfarm.kr -cmsxrootd-1.gridka.de -cmsxrootd-2.gridka.de -cmsxrootd.ihep.ac.cn -dc2-grid-64.brunel.ac.uk -ds-202-04-05.cr.cnaf.infn.it -ds-202-04-07.cr.cnaf.infn.it -ds-202-11-05.cr.cnaf.infn.it -ds-202-11-07.cr.cnaf.infn.it -eoscms-srv-b2.cern.ch -eoscms-srv-m1.cern.ch -eymir.grid.metu.edu.tr -gaexroot01.ciemat.es -gfe02.grid.hep.ph.ic.ac.uk -grid143.kfki.hu -grid-vo-cms.physik.rwth-aachen.de -grse001.inr.troitsk.ru -hask.csc.fi -hephyse.oeaw.ac.at -heplnx228.pp.rl.ac.uk -heplnx229.pp.rl.ac.uk -ingrid-se03.cism.ucl.ac.be -ingrid-se04.cism.ucl.ac.be -ingrid-se05.cism.ucl.ac.be -ingrid-se06.cism.ucl.ac.be -lcgse01.phy.bris.ac.uk -lcgsexrd.jinr.ru -llrxrd-redir.in2p3.fr -maite.iihe.ac.be -node12.datagrid.cea.fr -nute.csc.fi -polgrid4.in2p3.fr -pool01.ifca.es -pool02.ifca.es -pool03.ifca.es -pool04.ifca.es -pool05.ifca.es -pool06.ifca.es -pool07.ifca.es -pool08.ifca.es -sbgse1.in2p3.fr -se01.grid.nchc.org.tw -se01.indiacms.res.in -se3.itep.ru -seadmin.ihep.ac.cn -se.cis.gov.pl -se.grid.icm.edu.pl -se-xrd01.jinr-t1.ru -ss-01.recas.ba.infn.it -ss-02.recas.ba.infn.it -ss-03.recas.ba.infn.it -stormgf1.pi.infn.it -stormgf2.pi.infn.it -stormgf3.pi.infn.it -t2-cms-xrootd01.desy.de -t2-cms-xrootd02.desy.de -t2-cms-xrootd03.desy.de -t2-xrdcms.lnl.infn.it -vobox0002.m45.ihep.su -xrootd01-cmst1.pic.es -xrootd01.grid.hep.ph.ic.ac.uk -xrootd02-cmst1.pic.es -xrootd02.grid.hep.ph.ic.ac.uk -xrootd1.hep.kbfi.ee -xrootd2.hep.kbfi.ee -xrootd3.hep.kbfi.ee -xroot-data01.ncg.ingrid.pt -xroot-data02.ncg.ingrid.pt -xrootd.ba.infn.it -xrootd-cms-01.cr.cnaf.infn.it -xrootd-cms-02.cr.cnaf.infn.it -xrootd-cms-uk.gridpp.rl.ac.uk -xrootd-es-cie.ciemat.es -xrootd-es-pic.pic.es -xroot.ncg.ingrid.pt -cms-b005.rcac.purdue.edu -cms-b006.rcac.purdue.edu -cms-b007.rcac.purdue.edu -cms-b008.rcac.purdue.edu -cms-b009.rcac.purdue.edu -cms-b012.rcac.purdue.edu -cms-c000.rcac.purdue.edu -cms-c001.rcac.purdue.edu -cms-c002.rcac.purdue.edu -cms-c003.rcac.purdue.edu -cms-c004.rcac.purdue.edu -cms-c005.rcac.purdue.edu -cms-c006.rcac.purdue.edu -cms-c007.rcac.purdue.edu -cms-c008.rcac.purdue.edu -cms-c009.rcac.purdue.edu -cms-d000.rcac.purdue.edu -cms-d001.rcac.purdue.edu -cms-f000.rcac.purdue.edu -cms-f001.rcac.purdue.edu -cms-f002.rcac.purdue.edu -cms-f003.rcac.purdue.edu -cms-g000.rcac.purdue.edu -cms-g001.rcac.purdue.edu -cms-g002.rcac.purdue.edu -cms-g003.rcac.purdue.edu -cms-g004.rcac.purdue.edu -cms-i000.rcac.purdue.edu -cms-i001.rcac.purdue.edu -cms-i002.rcac.purdue.edu -cms-i003.rcac.purdue.edu -cms-i004.rcac.purdue.edu -cms-i005.rcac.purdue.edu -cms-i006.rcac.purdue.edu -cmsio10.rc.ufl.edu -cmsio3.rc.ufl.edu -cmsio4.rc.ufl.edu -cmsio5.rc.ufl.edu -cmsio6.rc.ufl.edu -cmsio7.rc.ufl.edu -cmsio8.rc.ufl.edu -cmsio9.rc.ufl.edu -cms-j000.rcac.purdue.edu -cms-j001.rcac.purdue.edu -cms-j002.rcac.purdue.edu -cms-j003.rcac.purdue.edu -cms-k000.rcac.purdue.edu -cms-k001.rcac.purdue.edu -cms-k002.rcac.purdue.edu -cms-k003.rcac.purdue.edu -cms-k004.rcac.purdue.edu -cms-k005.rcac.purdue.edu -cms-k006.rcac.purdue.edu -cms-k007.rcac.purdue.edu -cms-k008.rcac.purdue.edu -cms-k009.rcac.purdue.edu -cms-l000.rcac.purdue.edu -cms-l001.rcac.purdue.edu -cms-l002.rcac.purdue.edu -cms-m000.rcac.purdue.edu -cms-m001.rcac.purdue.edu -cmsxrootd-site1.fnal.gov -cmsxrootd-site2.fnal.gov -cmsxrootd-site3.fnal.gov -g22n01.hep.wisc.edu -g22n02.hep.wisc.edu -g22n03.hep.wisc.edu -g22n04.hep.wisc.edu -g22n05.hep.wisc.edu -g22n06.hep.wisc.edu -g22n07.hep.wisc.edu -g22n08.hep.wisc.edu -g22n09.hep.wisc.edu -g22n10.hep.wisc.edu -g22n11.hep.wisc.edu -g22n12.hep.wisc.edu -g22n13.hep.wisc.edu -g22n15.hep.wisc.edu -g22n16.hep.wisc.edu -g22n17.hep.wisc.edu -gftp-1.t2.ucsd.edu -osg-se.sprace.org.br -pubxrootd.hep.wisc.edu -red-gridftp10.unl.edu -red-gridftp11.unl.edu -red-gridftp12.unl.edu -red-gridftp1.unl.edu -red-gridftp2.unl.edu -red-gridftp3.unl.edu -red-gridftp4.unl.edu -red-gridftp5.unl.edu -red-gridftp6.unl.edu -red-gridftp7.unl.edu -red-gridftp8.unl.edu -red-gridftp9.unl.edu -redirector.t2.ucsd.edu -s17n01.hep.wisc.edu -s17n02.hep.wisc.edu -s17n04.hep.wisc.edu -s17n05.hep.wisc.edu -s17n07.hep.wisc.edu -s17n08.hep.wisc.edu -s17n09.hep.wisc.edu -s17n10.hep.wisc.edu -se1.accre.vanderbilt.edu -se8.accre.vanderbilt.edu -sea.accre.vanderbilt.edu -seb.accre.vanderbilt.edu -sec.accre.vanderbilt.edu -sed.accre.vanderbilt.edu -sef.accre.vanderbilt.edu -seg.accre.vanderbilt.edu -seh.accre.vanderbilt.edu -spstrg04.sprace.org.br -spstrg05.sprace.org.br -spstrg06.sprace.org.br -spstrg07.sprace.org.br -spstrg08.sprace.org.br -spstrg09.sprace.org.br -test-003.t2.ucsd.edu -transfer-1.ultralight.org -transfer-2.ultralight.org -transfer-4.ultralight.org -transfer-5.ultralight.org -transfer-6.ultralight.org -transfer-7.ultralight.org -transfer-8.ultralight.org -xrootd10.cmsaf.mit.edu -xrootd11.cmsaf.mit.edu -xrootd12.cmsaf.mit.edu -xrootd13.cmsaf.mit.edu -xrootd14.cmsaf.mit.edu -xrootd15.cmsaf.mit.edu -xrootd16.cmsaf.mit.edu -xrootd17.cmsaf.mit.edu -xrootd18.cmsaf.mit.edu -xrootd9.cmsaf.mit.edu -xrootd.accre.vanderbilt.edu -xrootd.cmsaf.mit.edu -xrootd-local.unl.edu -xrootd.rcac.purdue.edu diff --git a/AAAOps/Federation/transit-hostIPv4.txt b/AAAOps/Federation/transit-hostIPv4.txt deleted file mode 100644 index 78913c7..0000000 --- a/AAAOps/Federation/transit-hostIPv4.txt +++ /dev/null @@ -1,17 +0,0 @@ -bonner03.rice.edu -bonner04.rice.edu -cms-se.hep.uprm.edu -cmsxrd.ts.infn.it -ds-t3-01.cr.cnaf.infn.it -ds-t3-02.cr.cnaf.infn.it -hepcms-0.umd.edu -hepxrd01.colorado.edu -kodiak-se.baylor.edu -lyogrid06.in2p3.fr -ntugrid4.phys.ntu.edu.tw -ntugrid6.phys.ntu.edu.tw -pcncp22.ncp.edu.pk -ruhex-osgce.rutgers.edu -storm.mib.infn.it -t3se01.psi.ch -xrootd.knu.ac.kr From 94b1fd958348dac6f43fef161b0d8d19609d2c37 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 25 Jun 2019 11:45:11 +0200 Subject: [PATCH 011/192] updated .gitignore file --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index a87d7df..6243879 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ -works/ AAAOps/Federation/in/ AAAOps/Federation/out/ AAAOps/tools/timestamp.sh -../Federation/hostIPv4.txt -../Federation/transit-hostIPv4.txt AAAOps/Federation/hostIPv4.txt AAAOps/Federation/transit-hostIPv4.txt From e413052f751641cdcf2e0e26b12f7c5a6f852cae Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 25 Jun 2019 11:48:42 +0200 Subject: [PATCH 012/192] deleted .create_fedmaps_DEV.py.swo pushed by mistake --- AAAOps/Federation/.create_fedmaps_DEV.py.swo | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 AAAOps/Federation/.create_fedmaps_DEV.py.swo diff --git a/AAAOps/Federation/.create_fedmaps_DEV.py.swo b/AAAOps/Federation/.create_fedmaps_DEV.py.swo deleted file mode 100644 index 9be830c2060aadccf57e2db26842a9081192b8f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNO^h5z6)uP0jcq`Q#3c$U&mwiNwC>qmBUv745;op7UddjAJ+^|zlTOcc%}(1h zU7hNh{gJgYLIMey0|!6?!_5+0FcLTH9Jp{wjzmhhAz?0v6PH**BHyd3p6OZJ80VBo z-O@MRUGLRA ziBmyOs5Iyeo}^!wnXb1~l;(Y*S|S?NS65drWO@CJh=q!DBGYA`; zl_1-DlEJFIVxVH+85lTv`NYcd5_Oob9pf*)aQPYfR;m>P6$2Fm6$2Fm6$2Fm6$2Fm z6$77k26T3a-NYcCa|5~N?uQQC-D`EL7^oPi7^oPi7^oPi7^oPi7^oPi7^oPi7^oQd z6fzLSjQt96ze)c9fdBt5{{Dae%Z&XJ_z}1#h4}2F`1$^M)VaEOp{2urj@IBx%&;Y&){Ox(h{si0t zTEIH61}p<#1O79|*ayHJ;0HhkNZ^~mtH3Fs20Rb^_lt-Xcn7!-+yj0Md<$3xmVlRl zBf!7E06pMs;688jQtMyAd36eY@7v3WGcDw~1 z<-FfuY>v!Yf9+bPWL)b6Q3emOUu&&|XD)`PPj76leEURgfz2g-j`J*+5G`*ud9Afv zmh&i$C*mY6jCy3$>+l96PsLSuOy$l%C^2>F?AzgzHhiX1{}03D_6ZuAPSkiE>$)#h z*-DO4^m)P zj5jFu$UO5TH!)_zs3&5WBdQ z$>MAxGKKLKyig}=yyOnkkA_3CmDA@hQTFNCPo_DdaIB$((KB5{M?xcw9K?$KkGCS>lcIi{%c)PdVxH*JF$(#OeMVf*^DUu^{tOc%42h#KG|My$ zDFI7$;=@6(hc|{8XG-Z4!qd51`u)v;iQclde zYV8U&F$Uws++*{}D3eN~86}8_sj8Mu-@X)T| zN2*!}J}7&rF=PXcO-wNx6JIZqMtg^F1>MXZ^uh$B%79|ZwAzD^&2xH~jECygj^Kj` zRW+ie+Z~FL-Ow&yJQvL5%q+F|>}GDFphzYy#pX0jQWqqNVcTnXR6?k|VSW)5y|f$j zYo{`F7HpPXCn}gj?54(AgZE^rlXNT?iD^PoyQyK3j)FWJf==&h>bVVL5)@puJ7!iE zMIuHAniFKGG!f~wxX|1t_BJtnrpvAvUoO;k(lbJvi=taeK*_RdL8P@xx_0rFCNp)S zaX2*^ihInb+rWW9Y}ekqob5u3O@TORKV5f?|AtwcsDt}F4@SmYYVlIu_;*Ep%%jgtfsP|2Rwh}o4vi-jRJP7ic% z^069T=w5R>Eu%Ku;$*|5yAq(qd}c)IBCnb6hg$JA`9>t^K2^Y(%Na3kjk2X62xg7BhO#|s+CON~ zrrkPeaLOuwZEa)Ypc<0Vl9?RJEfi+dcbq5haV>VW?P5|(2}D~oT14-yq3q(bb{k(j z4fB Date: Tue, 25 Jun 2019 13:22:13 +0200 Subject: [PATCH 013/192] removed not needed parsing fix --- AAAOps/Federation/create_allow-list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/Federation/create_allow-list.sh b/AAAOps/Federation/create_allow-list.sh index bff1d1d..63462b6 100755 --- a/AAAOps/Federation/create_allow-list.sh +++ b/AAAOps/Federation/create_allow-list.sh @@ -38,7 +38,7 @@ for j in "${redirectors[@]}";do done else xrdmapc --list all "$j" | tail -n +2 | awk '{if($2=="Man") print $3; else print $2}' > $BASE/tmp_total - cat $BASE/tmp_total | cut -d : -f1 | grep -v "\[" | sort -u > $FEDINFO/in/trans.txt + cat $BASE/tmp_total | cut -d : -f1 | sort -u > $FEDINFO/in/trans.txt rm transit-hostIPv4.txt transit-hostIPv6.txt for f in $(cat $FEDINFO/in/trans.txt);do From b9dc66351196df80f31d76a0f08ebc4e47d231a3 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Thu, 27 Jun 2019 09:40:48 +0200 Subject: [PATCH 014/192] modified git ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6243879..03e7ade 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ AAAOps/Federation/out/ AAAOps/tools/timestamp.sh AAAOps/Federation/hostIPv4.txt AAAOps/Federation/transit-hostIPv4.txt +hostIPv4.txt +transit-hostIPv4.txt From 869104125a3ca53c75ff6e530e79f8b2e23971e5 Mon Sep 17 00:00:00 2001 From: Pradeep Date: Tue, 6 Aug 2019 11:18:59 -0500 Subject: [PATCH 015/192] updated 2019 pledge values and changed site usable space of CERN_MSS to 0 --- storage_overview/overview.py | 48 +++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/storage_overview/overview.py b/storage_overview/overview.py index fdc4d6c..388c5cc 100755 --- a/storage_overview/overview.py +++ b/storage_overview/overview.py @@ -54,22 +54,22 @@ ) # TB PLEDGES = { - "T0_CH_CERN_MSS": 70500, #2016 pledge 44000 - "T1_DE_KIT_MSS": 19500, #2016 pledge 10000 - "T1_ES_PIC_MSS": 6885, #2016 pledge 5100 - "T1_FR_CCIN2P3_MSS": 11000, #2016 pledge 8100 - "T1_IT_CNAF_MSS": 21000, #early 2017 16000 #2016 pledge 12000 - "T1_RU_JINR_MSS": 8000, #2016 pledge 5000 - "T1_UK_RAL_MSS": 12720, #2016 pledge 8000 - "T1_US_FNAL_MSS": 54000, #2016 pledge 40000 - "T1_DE_KIT_Disk": 5140, #until early 2017 3500 - "T1_ES_PIC_Disk": 2499, #until early 2017 1683 - "T1_FR_CCIN2P3_Disk": 3600, #until early 2017 2700 - "T1_IT_CNAF_Disk": 5880, #until early 2017 3960 - "T1_RU_JINR_Disk": 3900, #early 2017 4000 #2800,# SE(2800TB) + cache for tapes(400TB) - "T1_UK_RAL_Disk": 4304, #early 2027 2640 - "T1_US_FNAL_Disk": 19600, # pledge 13200 until April 1 2017 - "T2_CH_CERN": 24600 # early 2017 5370 + "T0_CH_CERN_MSS": 97000, #2017 pledge70500, #2016 pledge 44000 + "T1_DE_KIT_MSS": 18800, #2017 pledge 19500, #2016 pledge 10000 + "T1_ES_PIC_MSS": 8629, #2017 pledge 6885, #2016 pledge 5100 + "T1_FR_CCIN2P3_MSS": 14500,#2017 pledge 11000, #2016 pledge 8100 + "T1_IT_CNAF_MSS": 24440, #2017 pledge 21000, #early 2017 16000 #2016 pledge 12000 + "T1_RU_JINR_MSS": 9000, #2017 pledge 8000, #2016 pledge 5000 + "T1_UK_RAL_MSS": 14936, #2017 pledge 12720, #2016 pledge 8000 + "T1_US_FNAL_MSS": 75200, #2017 pledge 54000, #2016 pledge 40000 + "T1_DE_KIT_Disk": 6000, #2017 pledge 5140, #until early 2017 3500 + "T1_ES_PIC_Disk": 2754, #2017 pledge 2499, #until early 2017 1683 + "T1_FR_CCIN2P3_Disk": 4600, #2017 pledge 3600, #until early 2017 2700 + "T1_IT_CNAF_Disk": 7200, #2017 pledge 5880, #until early 2017 3960 + "T1_RU_JINR_Disk": 6000, #2017 pledge 3900, #early 2017 4000 #2800,# SE(2800TB) + cache for tapes(400TB) + "T1_UK_RAL_Disk": 4784, #2017 pledge 4304, #early 2027 2640 + "T1_US_FNAL_Disk": 24000, #2017 pledge 19600, # pledge 13200 until April 1 2017 + "T2_CH_CERN": 26100, #2017 pledge 24600 # early 2017 5370 } T0CERNPLEDGE=44000 T0CERNUSED=31524.8*UNIT @@ -400,7 +400,10 @@ def meetingSummary(self, isTape=False): #row.append(self._create_cell(to_TB(PLEDGES[FNALTAPE]*UNIT - T1FNALMSSUSED))) #grand_total += T1FNALMSSUSED ''' - if site.name == "T0_CH_CERN_MSS" or site.name == "T1_DE_KIT_MSS" or site.name == "T1_DE_KIT_Disk" or site.name == "T1_IT_CNAF_MSS" or site.name == "T1_IT_CNAF_Disk": + #sites_set_unusable = ["T0_CH_CERN_MSS", "T1_IT_CNAF_Disk", "T1_IT_CNAF_MSS", "T1_DE_KIT_Disk", "T1_DE_KIT_MSS"] + sites_set_unusable = [] + + if site.name in sites_set_unusable: row.append(self._create_cell(to_TB(0))) else: row.append(self._create_cell(to_TB(total))) @@ -444,13 +447,12 @@ def jsonSummary(self, isTape=False): jsonsummary['Free'][site.name] = int((PLEDGES[site.name]*UNIT - T1FNALMSSUSED)/UNIT) for site in jsonsummary['Free']: - ''' - if (site == 'T1_US_FNAL_MSS' or site =='T1_DE_KIT_MSS'): - jsonsummary['Usable'][site] = 0 - ''' - if (site == 'T0_CH_CERN_MSS'): + #sites_set_unusable = ["T0_CH_CERN_MSS", "T1_IT_CNAF_Disk", "T1_IT_CNAF_MSS", "T1_DE_KIT_Disk", "T1_DE_KIT_MSS"] + sites_set_unusable = ["T0_CH_CERN_MSS"] + + if site in sites_set_unusable: jsonsummary['Usable'][site] = 0 - else: + else: jsonsummary['Usable'][site]=jsonsummary['Free'][site] #uncoment this part when there is no blacklisting to any site From e6e3f11aaca75f79e38d30507c4b5dd1914b4b74 Mon Sep 17 00:00:00 2001 From: Donata Mielaikaite Date: Fri, 20 Sep 2019 14:18:52 +0200 Subject: [PATCH 016/192] fix csv2xml parsing for inject api --- scripts/csv2xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/csv2xml.py b/scripts/csv2xml.py index 422b4b5..af10a1a 100644 --- a/scripts/csv2xml.py +++ b/scripts/csv2xml.py @@ -31,7 +31,7 @@ block = ET.SubElement(dataset,"block", name=key2) block.set("is-open", "n") for subitem in items2: - file = ET.SubElement(block,"file",name=subitem[2],bytes=subitem[3],checksum=subitem[4]+","+subitem[5]) + file = ET.SubElement(block,"file",name=subitem[2],bytes=subitem[3],checksum='cksum:'+subitem[4]+",adler32:"+subitem[5]) outfile = open(sys.argv[1]+'.xml', 'w') outfile.writelines(minidom.parseString(ET.tostring(root)).toprettyxml(indent = " ")) From 41f0e3eaefd77e46aa40d40ac095dbe0db366e0f Mon Sep 17 00:00:00 2001 From: Donata Mielaikaite Date: Tue, 24 Sep 2019 13:54:19 +0200 Subject: [PATCH 017/192] added script helper on daily work --- scripts/find_missing_datasets.py | 41 ++++++++++++++++++++++++++++++++ scripts/get_routed_bytes.py | 13 ++++++---- 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100755 scripts/find_missing_datasets.py diff --git a/scripts/find_missing_datasets.py b/scripts/find_missing_datasets.py new file mode 100755 index 0000000..a168757 --- /dev/null +++ b/scripts/find_missing_datasets.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +import sys +import os +import json +import urllib2 + +## This tools uses Phedex API blockarrive +## USAGE: python find_missing_datasets.py + +def get_missing_data(): + url='https://cmsweb.cern.ch/phedex/datasvc/json/prod/nodes' + result = url_reading(url) + sites = get_node_name(result) + for site in sites: + get_block_with_no_source_replica(site) + +def get_node_name(result): + sites_list = []; + for node in result['phedex']['node']: + if "T1" in node['name'] or "T2" in node['name']: + sites_list.append(node['name']) + return sites_list + +def get_block_with_no_source_replica(site): + url='https://cmsweb.cern.ch/phedex/datasvc/json/prod/blockarrive?&basis=-6&to_node=' + site + result = url_reading(url) + with open('missing_data.txt', 'a+') as f: + f.write("%s\r\n" %site) + for block in result['phedex']['block']: + f.write("%s\r\n" %block['dataset']) + +def url_reading(url): + jstr = urllib2.urlopen(url).read() + jstr = jstr.replace("\n", " ") + result = json.loads(jstr) + return result + +def init(): + get_missing_data() + +init() diff --git a/scripts/get_routed_bytes.py b/scripts/get_routed_bytes.py index 988e930..6e02746 100644 --- a/scripts/get_routed_bytes.py +++ b/scripts/get_routed_bytes.py @@ -6,7 +6,7 @@ ## Get the volume routed to a site given the CMS site name + Phedex queue (priority) ## This tools uses Phedex API routedblocks -## USAGE: python get_routed_bytes.py +## USAGE: python get_routed_bytes.py def formatSize(size): output = '' @@ -24,20 +24,25 @@ def formatSize(size): output += "%.3f PB" % (float(size)/1E15) return output -def get_routed_bytes(site, priority): +def get_routed_bytes(site): url='https://cmsweb.cern.ch/phedex/datasvc/json/prod/routedblocks?to=' + site jstr = urllib2.urlopen(url).read() jstr = jstr.replace("\n", " ") result = json.loads(jstr) + all_priorities = ["low", "normal", "high", "reserved"] bytes = 0 + for priority in all_priorities: + get_priority(result,priority,bytes) +def get_priority(result,priority,bytes): for route in result['phedex']['route']: if (route['priority'] == priority): for block in route['block']: bytes += float(block['route_bytes']) - print formatSize(bytes) + final_record = priority + " "+ formatSize(bytes) + print final_record def init(): - get_routed_bytes(str(sys.argv[1]), str(sys.argv[2])) + get_routed_bytes(str(sys.argv[1])) init() From b75da305e71a2bc783b83d4040e82523efe778b7 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 26 Nov 2019 16:35:55 +0100 Subject: [PATCH 018/192] aaa subscription metric --- AAAOps/AAA_Subscription/aaa_federation.py | 77 ++++++++++++++ AAAOps/AAA_Subscription/aaa_federation.sh | 6 ++ AAAOps/AAA_Subscription/aaa_schema.json | 116 ++++++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100755 AAAOps/AAA_Subscription/aaa_federation.py create mode 100755 AAAOps/AAA_Subscription/aaa_federation.sh create mode 100644 AAAOps/AAA_Subscription/aaa_schema.json diff --git a/AAAOps/AAA_Subscription/aaa_federation.py b/AAAOps/AAA_Subscription/aaa_federation.py new file mode 100755 index 0000000..2d0dbd4 --- /dev/null +++ b/AAAOps/AAA_Subscription/aaa_federation.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +import os +import sys +import time +import subprocess +import urllib +import json +from datetime import datetime +import requests +import collections +import stomp +import argparse +from CMSMonitoring.StompAMQ import StompAMQ +import uuid +import hashlib +import stomp +from uuid import uuid4 +import time + + +class OptionParser(): + def __init__(self): + "User based option parser" + self.parser = argparse.ArgumentParser() + msg = "Send aaa subscription json file via StompAMQ to a broker, provide broker credentials in JSON file" + self.parser.add_argument("--amq", action="store", + dest="amq", default="", help=msg, required=True) + msg = " provide data json file" + self.parser.add_argument("--data", action="store", + dest="subscription_data", default="", help=msg, required=True) + + +def credentials(fname=None): + "Read credentials from MONIT_BROKER environment" + if not fname: + fname = os.environ.get('MONIT_BROKER', '') + if not os.path.isfile(fname): + raise Exception( + "Unable to locate MONIT credentials, please setup MONIT_BROKER") + return {} + with open(fname, 'r') as istream: + data = json.load(istream) + return data + + +if __name__ == "__main__": + optmgr = OptionParser() + opts = optmgr.parser.parse_args() + if opts.subscription_data: + try: + with open(opts.subscription_data, 'r') as f: + payload = json.load(f) + except IOError: + print("File not accessible") + + if opts.amq: + creds = credentials(opts.amq) + username = creds['username'] + password = creds['password'] + topic = creds['topic'] + host, port = creds['host_and_ports'].split(':') + port = int(port) + ckey = os.path.join("/root/pradeep/send_aaa_matrics/", 'userkey.pem') + cert = os.path.join("/root/pradeep/send_aaa_matrics/", 'usercert.pem') + producer = creds['producer'] + topic = creds['topic'] + if creds and StompAMQ: + amq = StompAMQ(username, password, producer, topic, key=ckey, + cert=cert, validation_schema=None, host_and_ports=[(host, port)]) + data = [] + for doc in payload: + hid = int(uuid4()) + notification, _, _ = amq.make_notification(doc, hid) + data.append(notification) + results = amq.send(data) + print("results", results) diff --git a/AAAOps/AAA_Subscription/aaa_federation.sh b/AAAOps/AAA_Subscription/aaa_federation.sh new file mode 100755 index 0000000..261d01f --- /dev/null +++ b/AAAOps/AAA_Subscription/aaa_federation.sh @@ -0,0 +1,6 @@ +#!/bin/bash + + +set -x +export PYTHONPATH=$PYTHONPATH:/root/pradeep/send_aaa_matrics/CMSMonitoring/src/python/ +python $PWD/test.py --amq /root/pradeep/send_aaa_matrics/credentials.json --data $PWD/aaa_schema.json diff --git a/AAAOps/AAA_Subscription/aaa_schema.json b/AAAOps/AAA_Subscription/aaa_schema.json new file mode 100644 index 0000000..f4ea273 --- /dev/null +++ b/AAAOps/AAA_Subscription/aaa_schema.json @@ -0,0 +1,116 @@ +{ + "T0_CH_CERN": "nowhere", + "T0_CH_CSCS_HPC": "nowhere", + "T1_DE_KIT": "prod", + "T1_ES_PIC": "prod", + "T1_FR_CCIN2P3": "prod", + "T1_IT_CNAF": "prod", + "T1_RU_JINR": "prod", + "T1_UK_RAL": "prod", + "T1_US_FNAL": "prod", + "T2_AT_Vienna": "prod", + "T2_BE_IIHE": "prod", + "T2_BE_UCL": "prod", + "T2_BR_SPRACE": "prod", + "T2_BR_UERJ": "prod", + "T2_CH_CERN": "prod", + "T2_CH_CERN_HLT": "nowhere", + "T2_CH_CSCS": "prod", + "T2_CN_Beijing": "prod", + "T2_DE_DESY": "prod", + "T2_DE_RWTH": "prod", + "T2_EE_Estonia": "prod", + "T2_ES_CIEMAT": "prod", + "T2_ES_IFCA": "prod", + "T2_FI_HIP": "prod", + "T2_FR_CCIN2P3": "prod", + "T2_FR_GRIF_IRFU": "prod", + "T2_FR_GRIF_LLR": "prod", + "T2_FR_IPHC": "prod", + "T2_GR_Ioannina": "prod", + "T2_HU_Budapest": "prod", + "T2_IN_TIFR": "prod", + "T2_IT_Bari": "prod", + "T2_IT_Legnaro": "prod", + "T2_IT_Pisa": "prod", + "T2_IT_Rome": "prod", + "T2_KR_KISTI": "prod", + "T2_PK_NCP": "trans", + "T2_PL_Swierk": "prod", + "T2_PL_Warsaw": "prod", + "T2_PT_NCG_Lisbon": "prod", + "T2_RU_IHEP": "prod", + "T2_RU_INR": "prod", + "T2_RU_ITEP": "prod", + "T2_RU_JINR": "prod", + "T2_TR_METU": "prod", + "T2_TW_NCHC": "prod", + "T2_UA_KIPT": "prod", + "T2_UK_London_Brunel": "prod", + "T2_UK_London_IC": "prod", + "T2_UK_SGrid_Bristol": "prod", + "T2_UK_SGrid_RALPP": "prod", + "T2_US_Caltech": "prod", + "T2_US_Florida": "prod", + "T2_US_MIT": "prod", + "T2_US_Nebraska": "prod", + "T2_US_Purdue": "prod", + "T2_US_UCSD": "prod", + "T2_US_Vanderbilt": "trans", + "T2_US_Wisconsin": "prod", + "T3_BG_UNI_SOFIA": "trans", + "T3_BY_NCPHEP": "nowhere", + "T3_CH_CERN_DOMA": "nowhere", + "T3_CH_CERN_OpenData": "nowhere", + "T3_CH_CMSAtHome": "nowhere", + "T3_CH_PSI": "trans", + "T3_CN_PKU": "nowhere", + "T3_FR_IPNL": "trans", + "T3_HR_IRB": "nowhere", + "T3_HU_Debrecen": "nowhere", + "T3_IN_TIFRCloud": "nowhere", + "T3_IN_VBU": "nowhere", + "T3_IR_IPM": "nowhere", + "T3_IT_Bologna": "trans", + "T3_IT_MIB": "trans", + "T3_IT_Perugia": "nowhere", + "T3_IT_Trieste": "trans", + "T3_KR_KISTI": "nowhere", + "T3_KR_KNU": "trans", + "T3_KR_UOS": "nowhere", + "T3_MX_Cinvestav": "nowhere", + "T3_RU_FIAN": "nowhere", + "T3_RU_MEPhI": "nowhere", + "T3_TH_CHULA": "nowhere", + "T3_TW_NCU": "nowhere", + "T3_TW_NTU_HEP": "trans", + "T3_UK_London_QMUL": "nowhere", + "T3_UK_London_RHUL": "nowhere", + "T3_UK_SGrid_Oxford": "nowhere", + "T3_UK_ScotGrid_GLA": "nowhere", + "T3_US_Baylor": "trans", + "T3_US_Brown": "trans", + "T3_US_Colorado": "trans", + "T3_US_FIT": "nowhere", + "T3_US_FNALLPC": "nowhere", + "T3_US_HEPCloud": "nowhere", + "T3_US_MIT": "nowhere", + "T3_US_Minnesota": "nowhere", + "T3_US_NERSC": "nowhere", + "T3_US_NotreDame": "nowhere", + "T3_US_OSG": "nowhere", + "T3_US_OSU": "nowhere", + "T3_US_Princeton_ICSE": "nowhere", + "T3_US_PuertoRico": "nowhere", + "T3_US_Rice": "trans", + "T3_US_Rutgers": "trans", + "T3_US_SDSC": "nowhere", + "T3_US_TAMU": "nowhere", + "T3_US_TTU": "nowhere", + "T3_US_UCR": "nowhere", + "T3_US_UMD": "trans", + "T3_US_UMiss": "nowhere", + "T3_US_UVA": "nowhere", + "T3_US_VC3_NotreDame": "nowhere", + "T3_US_Vanderbilt_EC2": "prod" +} \ No newline at end of file From 74c8c44450599986a8472eced89a2874c201b119 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 26 Nov 2019 10:21:26 -0600 Subject: [PATCH 019/192] Update aaa_federation.py --- AAAOps/AAA_Subscription/aaa_federation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/AAA_Subscription/aaa_federation.py b/AAAOps/AAA_Subscription/aaa_federation.py index 2d0dbd4..be408eb 100755 --- a/AAAOps/AAA_Subscription/aaa_federation.py +++ b/AAAOps/AAA_Subscription/aaa_federation.py @@ -61,8 +61,8 @@ def credentials(fname=None): topic = creds['topic'] host, port = creds['host_and_ports'].split(':') port = int(port) - ckey = os.path.join("/root/pradeep/send_aaa_matrics/", 'userkey.pem') - cert = os.path.join("/root/pradeep/send_aaa_matrics/", 'usercert.pem') + ckey = os.path.join("/root/.globus/", 'userkey.pem') + cert = os.path.join("/root/.globus/", 'usercert.pem') producer = creds['producer'] topic = creds['topic'] if creds and StompAMQ: From bf06583b073626afc5263f82191e09cf9b7b0bf2 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 26 Nov 2019 17:39:06 +0100 Subject: [PATCH 020/192] credentials from MONIT broker environment --- AAAOps/AAA_Subscription/credentials.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 AAAOps/AAA_Subscription/credentials.json diff --git a/AAAOps/AAA_Subscription/credentials.json b/AAAOps/AAA_Subscription/credentials.json new file mode 100755 index 0000000..65d3dfa --- /dev/null +++ b/AAAOps/AAA_Subscription/credentials.json @@ -0,0 +1,7 @@ +{ + "producer" : "cms_aaa_subscription", + "type" : "metric", + "type_prefix" : "raw", + "topic" : "cms.aaa.subscription", + "host_and_ports" : "cms-mb.cern.ch:61313", +} From 45de02e266ac34dd09d0137b5f3e4d7d4cad6cd7 Mon Sep 17 00:00:00 2001 From: Pradeep Date: Mon, 9 Dec 2019 16:13:26 -0600 Subject: [PATCH 021/192] spark DBS3 and Phedex consistency run --- consistency_check/spark/README.md | 19 + .../spark/dbs_phedex_consistency.py | 113 ++++ consistency_check/spark/filemismatch.py | 55 ++ consistency_check/spark/schemas.py | 523 ++++++++++++++++++ consistency_check/spark/setup.sh | 5 + 5 files changed, 715 insertions(+) create mode 100644 consistency_check/spark/README.md create mode 100644 consistency_check/spark/dbs_phedex_consistency.py create mode 100644 consistency_check/spark/filemismatch.py create mode 100644 consistency_check/spark/schemas.py create mode 100644 consistency_check/spark/setup.sh diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md new file mode 100644 index 0000000..262037e --- /dev/null +++ b/consistency_check/spark/README.md @@ -0,0 +1,19 @@ +# Basic setup + + +login to lxplus7.cern.ch +run setup.sh + + + + + +## run dbs and phedex consistency check on spark: + +Example : python dbs_phedex_consistency.py --timestamp 2019-12-09 --dbs invalid --phedex present --output hdfs://analytix/user/pjasal/ + + + +## run file mismatch using +Example : +python filemismatch.py --timestamp 7 diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py new file mode 100644 index 0000000..02883b0 --- /dev/null +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -0,0 +1,113 @@ +#!/usr/bin/env spark-submit +from __future__ import print_function +import argparse +import json +import os +from datetime import datetime, timedelta + +from pyspark import SparkConf, SparkContext, StorageLevel +from pyspark.sql import SparkSession, Column +from pyspark.sql.functions import col +import pyspark.sql.functions as fna +import pyspark.sql.types as types +from pyspark.sql import DataFrame + + +# stolen from CMSSpark +import schemas + +class OptionParser(): + def __init__(self): + "option Parser" + self.parser = argparse.ArgumentParser(prog='consistency') + msg = "Optional required timestamp format yyyy-mm-dd , example : 2019-01-31 " + self.parser.add_argument("--timestamp",action="store",dest="phedex_time",default=None,help=msg) + msg = "phedex dataset option: it takes 'missing' or 'present'" + self.parser.add_argument("--phedex",action="store",dest="phedex_status",default=None,help=msg,required=True) + msg = "DBS dataset status" + self.parser.add_argument("--dbs",action="store",dest="dbs_status",default=None,help=msg,required=True) + msg= "Output path in HDFS for result" + self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) + + + +def run_consistency(args): + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + spark = SparkSession(sc) + print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) + phedex_time_stamp = valid_data(args.phedex_time) + + avroreader = spark.read.format("com.databricks.spark.avro") + csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") + dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") + dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") + dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + + if args.phedex_status == 'present' and args.dbs_status == 'invalid': + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" + phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) + ''' + for reference dbs d_dataset_access_type_id: + 1 : valid + 2 : invalid + 42 : Deprecated + 41 : Production + 81 : Deleted + ''' + out = args.out_path + "/invalid_dbs_present_phedex" + working_set = (dbs_datasets + .filter(col('d_dataset_access_type_id')=='2') + .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('dataset_name').isNotNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info + .distinct()) + working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) + working_set.groupby("input_campaign").agg((fn.count(fn.col("dataset_name").isNotNull())).show() + elif args.phedex_status == 'present' and args.dbs_status == 'deleted': + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" + phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) + out = args.out_path + "/deleted_dbs_present_phedex" + working_set = (dbs_datasets + .filter(col('d_dataset_access_type_id')=='81') + .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('dataset_name').isNotNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','d_dataset') # you can select more columns for detail info + .distinct()) + working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) + working_set.groupby("input_campaign").agg((fn.count(fn.col("dataset_name").isNotNull())).show() + elif args.phedex_status == 'missing' and args.dbs_status == 'valid': + # to be on the safe side : adding time difference for block replicas injected + d = phedex_time_stamp - timedelta(days=30) # should add argumentparser option + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(d) + "_*/part-m-00000" + phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) + out = args.out_path + "/valid_dbs_missing_phedex" + working_set = (dbs_datasets + .filter(col('d_dataset_access_type_id')=='1') + .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('block_name').isNotNull() & col('node_id').isNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('block_name') + .distinct()) + working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) + working_set.groupby("input_campaign").agg((fn.count(fn.col("dataset_name").isNotNull())).show() + +def valid_data(args.phedex_time): + if args.phedex_time: + try : + return datetime.strptime(args.phedex_time,"%Y%m%d") + except ValueError: + msg = " Not a valid date : '{0}'.".format(args.phedex_time) + raise argparse.ArgumentTypeError(msg) + else: + return datetime.datetime.today().strftime("%Y%m%d") + +if __name__ == '__main__': + optmgr = OptionParser() + args = optmgr.parser.parse_args() + run_consistency(args) diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py new file mode 100644 index 0000000..ea624c5 --- /dev/null +++ b/consistency_check/spark/filemismatch.py @@ -0,0 +1,55 @@ +#!/usr/bin/env spark-submit +from __future__ import print_function +import argparse +import json +import os +from datetime import datetime, timedelta + +from pyspark import SparkConf, SparkContext, StorageLevel +from pyspark.sql import SparkSession, Column +from pyspark.sql.functions import col +import pyspark.sql.functions as fna +import pyspark.sql.types as types +from pyspark.sql import DataFrame + + +# stolen from CMSSpark +import schemas + +class OptionParser(): + def __init__(self): + "option Parser" + self.parser = argparse.ArgumentParser(prog='consistency') + msg = "Files mismatch since last n days : default in past 7 days" + self.parser.add_argument("--timestamp",action="store",dest="timestamp",default=7,help=msg,type=int) + +def fileMismatch(args): + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + spark = SparkSession(sc) + print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) + + avroreader = spark.read.format("com.databricks.spark.avro") + csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") + dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") + dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + + current = datetime.datetime.utcnow().timestamp() + past_n_days = args.timestamp + delta_t = current - past_n_days*60*60*24 + + if args.phedex_status == 'present' and args.dbs_status == 'invalid': + working_set = (dbs_files + .filter(col('f_is_file_valid')=='1') + .filter(col('f_last_modification_date') >= delta_t) + .join(dbs_datasets,col('f_dataset_id')==col('d_dataset_id')) + .filter(col('d_dataset_access_type_id')=='1') + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','d_dataset','d_last_modified_by','f_logical_file_name') # you can select more columns for detail info + .distinct()) + working_set.select('f_logical_file_name').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(out) + working_set.groupby('d_dataset').agg((fn.count(fn.col("f_logical_file_name").isNotNull())).show() +if __name__ == '__main__': + optmgr = OptionParser() + args = optmgr.parser.parse_args() + fileMismatch(args) diff --git a/consistency_check/spark/schemas.py b/consistency_check/spark/schemas.py new file mode 100644 index 0000000..c624c62 --- /dev/null +++ b/consistency_check/spark/schemas.py @@ -0,0 +1,523 @@ + +#!/usr/bin/env python +#-*- coding: utf-8 -*- +#pylint: disable= +# Author: Valentin Kuznetsov +""" +Schema module for DBS/PhEDEx/AAA/EOS/CMSSW/JobMonitoring meta-data on HDFS +""" + +# system modules +import os +import sys +import time +import json + +# spark modules +from pyspark.sql.types import DoubleType, IntegerType, StructType, StructField, StringType, BooleanType, LongType + +def schema_processing_eras(): + """ + DBS PROCESSING_ERAS table schema + PROCESSING_ERA_ID NOT NULL NUMBER(38) + PROCESSING_ERA_NAME NOT NULL VARCHAR2(120) + CREATION_DATE NOT NULL INTEGER + CREATE_BY NOT NULL VARCHAR2(500) + DESCRIPTION NOT NULL VARCHAR2(40) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("processing_era_id", IntegerType(), True), + StructField("processing_version", StringType(), True), + StructField("creation_date", IntegerType(), True), + StructField("create_by", StringType(), True), + StructField("description", StringType(), True) + ]) + +def schema_acquisition_eras(): + """ + DBS ACQUISITION_ERAS table schema + ACQUISITION_ERA_ID NOT NULL NUMBER(38) + ACQUISITION_ERA_NAME NOT NULL VARCHAR2(120) + START_DATE NOT NULL INTEGER + END_DATE NOT NULL INTEGER + CREATION_DATE NOT NULL INTEGER + CREATE_BY NOT NULL VARCHAR2(500) + DESCRIPTION NOT NULL VARCHAR2(40) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("acquisition_era_id", IntegerType(), True), + StructField("acquisition_era_name", StringType(), True), + StructField("start_date", IntegerType(), True), + StructField("end_date", IntegerType(), True), + StructField("creation_date", IntegerType(), True), + StructField("create_by", StringType(), True), + StructField("description", StringType(), True) + ]) + +def schema_dataset_access_types(): + """ + DBS DATASET_ACCESS_TYPES table schema + DATASET_ACCESS_TYPE_ID NOT NULL NUMBER(38) + DATASET_ACCESS_TYPE NOT NULL VARCHAR2(100) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("dataset_access_type_id", IntegerType(), True), + StructField("dataset_access_type", StringType(), True) + ]) + +def schema_data_tiers(): + """ + DBS DATA_TIERS table schema + DATA_TIER_ID NOT NULL NUMBER(38) + DATA_TIER_NAME NOT NULL VARCHAR2(100) + CREATION_DATE NOT NULL NUMBER(38) + CREATE_BY NOT NULL VARCHAR2(100) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("data_tier_id", IntegerType(), True), + StructField("data_tier_name", StringType(), True), + StructField("data_tier_creation_date", DoubleType(), True), + StructField("data_tier_create_by", StringType(), True) + ]) + +def schema_datasets(): + """ + DBS DATASETS table schema + DATASET_ID NOT NULL NUMBER(38) + DATASET NOT NULL VARCHAR2(700) + IS_DATASET_VALID NOT NULL NUMBER(38) + PRIMARY_DS_ID NOT NULL NUMBER(38) + PROCESSED_DS_ID NOT NULL NUMBER(38) + DATA_TIER_ID NOT NULL NUMBER(38) + DATASET_ACCESS_TYPE_ID NOT NULL NUMBER(38) + ACQUISITION_ERA_ID NUMBER(38) + PROCESSING_ERA_ID NUMBER(38) + PHYSICS_GROUP_ID NUMBER(38) + XTCROSSSECTION FLOAT(126) + PREP_ID VARCHAR2(256) + CREATION_DATE NUMBER(38) + CREATE_BY VARCHAR2(500) + LAST_MODIFICATION_DATE NUMBER(38) + LAST_MODIFIED_BY VARCHAR2(500) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("d_dataset_id", IntegerType(), True), + StructField("d_dataset", StringType(), True), + StructField("d_is_dataset_valid", IntegerType(), True), + StructField("d_primary_ds_id", IntegerType(), True), + StructField("d_processed_ds_id", IntegerType(), True), + StructField("d_data_tier_id", IntegerType(), True), + StructField("d_dataset_access_type_id", IntegerType(), True), + StructField("d_acquisition_era_id", IntegerType(), True), + StructField("d_processing_era_id", IntegerType(), True), + StructField("d_physics_group_id", IntegerType(), True), + StructField("d_xtcrosssection", DoubleType(), True), + StructField("d_prep_id", StringType(), True), + StructField("d_creation_date", DoubleType(), True), + StructField("d_create_by", StringType(), True), + StructField("d_last_modification_date", DoubleType(), True), + StructField("d_last_modified_by", StringType(), True) + ]) + +def schema_blocks(): + """ + DBS BLOCKS table schema + BLOCK_ID NOT NULL NUMBER(38) + BLOCK_NAME NOT NULL VARCHAR2(500) + DATASET_ID NOT NULL NUMBER(38) + OPEN_FOR_WRITING NOT NULL NUMBER(38) + ORIGIN_SITE_NAME NOT NULL VARCHAR2(100) + BLOCK_SIZE NUMBER(38) + FILE_COUNT NUMBER(38) + CREATION_DATE NUMBER(38) + CREATE_BY VARCHAR2(500) + LAST_MODIFICATION_DATE NUMBER(38) + LAST_MODIFIED_BY VARCHAR2(500) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("b_block_id", IntegerType(), True), + StructField("b_block_name", StringType(), True), + StructField("b_dataset_id", IntegerType(), True), + StructField("b_open_for_writing", IntegerType(), True), + StructField("b_origin_site_name", StringType(), True), + StructField("b_block_size", DoubleType(), True), + StructField("b_file_count", IntegerType(), True), + StructField("b_creation_date", DoubleType(), True), + StructField("b_create_by", StringType(), True), + StructField("b_last_modification_date", DoubleType(), True), + StructField("b_last_modified_by", StringType(), True) + ]) + +def schema_files(): + """ + DBS FILES table schema + FILE_ID NOT NULL NUMBER(38) + LOGICAL_FILE_NAME NOT NULL VARCHAR2(500) + IS_FILE_VALID NOT NULL NUMBER(38) + DATASET_ID NOT NULL NUMBER(38) + BLOCK_ID NOT NULL NUMBER(38) + FILE_TYPE_ID NOT NULL NUMBER(38) + CHECK_SUM NOT NULL VARCHAR2(100) + EVENT_COUNT NOT NULL NUMBER(38) + FILE_SIZE NOT NULL NUMBER(38) + BRANCH_HASH_ID NUMBER(38) + ADLER32 VARCHAR2(100) + MD5 VARCHAR2(100) + AUTO_CROSS_SECTION FLOAT(126) + CREATION_DATE NUMBER(38) + CREATE_BY VARCHAR2(500) + LAST_MODIFICATION_DATE NUMBER(38) + LAST_MODIFIED_BY VARCHAR2(500) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("f_file_id", IntegerType(), True), + StructField("f_logical_file_name", StringType(), True), + StructField("f_is_file_valid", IntegerType(), True), + StructField("f_dataset_id", IntegerType(), True), + StructField("f_block_id", IntegerType(), True), + StructField("f_file_type_id", IntegerType(), True), + StructField("f_check_sum", StringType(), True), + StructField("f_event_count", IntegerType(), True), + StructField("f_file_size", DoubleType(), True), + StructField("f_branch_hash_id", IntegerType(), True), + StructField("f_adler32", StringType(), True), + StructField("f_md5", StringType(), True), + StructField("f_auto_cross_section", DoubleType(), True), + StructField("f_creation_date", DoubleType(), True), + StructField("f_create_by", StringType(), True), + StructField("f_last_modification_date", DoubleType(), True), + StructField("f_last_modified_by", StringType(), True) + ]) + +def schema_mod_configs(): + """ + DBS DATASET_OUTPUT_MOD_CONFIGS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("mc_ds_output_mod_config_id", IntegerType(), True), + StructField("mc_dataset_id", IntegerType(), True), + StructField("mc_output_mod_config_id", IntegerType(), True) + ]) + +def schema_out_configs(): + """ + DBS OUTPUT_MODULE_CONFIGS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("oc_output_mod_config_id", IntegerType(), True), + StructField("oc_app_exec_id", IntegerType(), True), + StructField("oc_release_version_id", IntegerType(), True), + StructField("oc_parameter_set_hash_id", IntegerType(), True), + StructField("oc_output_module_label", StringType(), True), + StructField("oc_global_tag", StringType(), True), + StructField("oc_scenario", StringType(), True), + StructField("oc_creation_date", IntegerType(), True), + StructField("oc_create_by", StringType(), True) + ]) + +def schema_rel_versions(): + """ + DBS RELEASE_VERSIONS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("r_release_version_id", IntegerType(), True), + StructField("r_release_version", StringType(), True) + ]) + +def schema_file_lumis(): + """DBS FILE_LUMIS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("fl_run_num", IntegerType(), True), + StructField("fl_lumi_section_num", StringType(), True), + StructField("fl_file_id", IntegerType(), True) + ]) + +def schema_phedex_summary(): + """PhEDEx summary table schema + site,dataset,size,date,replica_date + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("date", LongType(), True), + StructField("site", StringType(), True), + StructField("dataset", StringType(), True), + StructField("size", LongType(), True), + StructField("replica_date", LongType(), True) + ]) + +def schema_phedex(): + """ + PhEDEx schema on HDFS + :returns: StructType consisting StructField array + """ + return StructType([StructField("now_sec", DoubleType(), True), + StructField("dataset_name", StringType(), True), + StructField("dataset_id", IntegerType(), True), + StructField("dataset_is_open", StringType(), True), + StructField("dataset_time_create", DoubleType(), True), + StructField("dataset_time_update", DoubleType(), True), + StructField("block_name", StringType(), True), + StructField("block_id", IntegerType(), True), + StructField("block_files", IntegerType(), True), + StructField("block_bytes", LongType(), True), + StructField("block_is_open", StringType(), True), + StructField("block_time_create", DoubleType(), True), + StructField("block_time_update", DoubleType(), True), + StructField("node_name", StringType(), True), + StructField("node_id", IntegerType(), True), + StructField("br_is_active", StringType(), True), + StructField("br_src_files", LongType(), True), + StructField("br_src_bytes", LongType(), True), + StructField("br_dest_files", LongType(), True), + StructField("br_dest_bytes", LongType(), True), + StructField("br_node_files", LongType(), True), + StructField("br_node_bytes", LongType(), True), + StructField("br_xfer_files", LongType(), True), + StructField("br_xfer_bytes", LongType(), True), + StructField("br_is_custodial", StringType(), True), + StructField("br_user_group_id", IntegerType(), True), + StructField("replica_time_create", DoubleType(), True), + StructField("replica_time_updater", DoubleType(), True)]) + +def schema_dbs_phedex(): + """ + Schema for DBS+PhEDEx aggregation, see dbs_phedex.py and adjust accordingly. + dataset_name,evts,size,date,dataset_access_type,acquisition_era_name,r_release_version,node_name,pbr_size,dataset_is_open,max_replica_time + "/14TeV_T1T1_2BC_350_100_MadGraph/Summer12-UpgrdStdGeom_DESIGN42_V17-v1/GEN",73528,4.4569181E7,1.343676821E9,VALID,DBS2_UNKNOWN_ACQUISION_ERA,CMSSW_4_2_8_SLHCstd2_patch2,T1_US_FNAL_Buffer,4.4569181E7,y,1.361368512E9 + """ + return StructType([ + StructField("dataset_name", StringType(), True), + StructField("evts", IntegerType(), True), + StructField("size", DoubleType(), True), + StructField("date", DoubleType(), True), + StructField("dataset_access_type", StringType(), True), + StructField("acquisition_era_name", StringType(), True), + StructField("r_release_version", StringType(), True), + StructField("node_name", StringType(), True), + StructField("pbr_size", DoubleType(), True), + StructField("dataset_is_open", StringType(), True), + StructField("max_replica_time", DoubleType(), True) + ]) + +def schema_cmssw(): + """Schema for CMSSW record + {"UNIQUE_ID":{"string":"08F8DD3A-0FFE-E611-B710-BC305B3909F1-1"},"FILE_LFN":{"string":"/store/data/Run2016F/JetHT/AOD/23Sep2016-v1/70000/D2B97318-A186-E611-A1EA-F8BC123BBE3C.root"},"FILE_SIZE":{"string":"3865077537"},"CLIENT_DOMAIN":{"string":"in2p3.fr"},"CLIENT_HOST":{"string":"sbgwn141"},"SERVER_DOMAIN":{"string":"in2p3.fr"},"SERVER_HOST":{"string":"sbgse20"},"SITE_NAME":{"string":"T2_FR_IPHC"},"READ_BYTES_AT_CLOSE":{"string":"438385807"},"READ_BYTES":{"string":"438385807"},"READ_SINGLE_BYTES":{"string":"8913451"},"READ_SINGLE_OPERATIONS":{"string":"19"},"READ_SINGLE_AVERAGE":{"string":"469129"},"READ_SINGLE_SIGMA":{"string":"1956390"},"READ_VECTOR_BYTES":{"string":"429472356"},"READ_VECTOR_OPERATIONS":{"string":"58"},"READ_VECTOR_AVERAGE":{"string":"7404700"},"READ_VECTOR_SIGMA":{"string":"6672770"},"READ_VECTOR_COUNT_AVERAGE":{"string":"37.4138"},"READ_VECTOR_COUNT_SIGMA":{"string":"35.242"},"FALLBACK":{"string":"-"},"USER_DN":{"string":"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=vmariani/CN=751637/CN=Valentina Mariani/CN=1516526926/CN=481221408/CN=1301887600/CN=1846615186/CN=2041527197"},"APP_INFO":{"string":"3809_https://glidein.cern.ch/3809/170228:163214:vmariani:crab:JetHT:Run2016F:DSm:4b_0"},"START_TIME":{"string":"1488325657"},"END_TIME":{"string":"1488326400"},"START_DATE":{"long":1488322057000},"END_DATE":{"long":1488322800000},"INSERT_DATE":{"long":1488323999000}} + """ + return StructType([ + StructField("UNIQUE_ID", StringType(), True), + StructField("FILE_LFN", StringType(), True), + StructField("FILE_SIZE", StringType(), True), + StructField("CLIENT_DOMAIN", StringType(), True), + StructField("CLIENT_HOST", StringType(), True), + StructField("SERVER_DOMAIN", StringType(), True), + StructField("SERVER_HOST", StringType(), True), + StructField("SITE_NAME", StringType(), True), + StructField("READ_BYTES_AT_CLOSE", StringType(), True), + StructField("READ_BYTES", StringType(), True), + StructField("READ_SINGLE_BYTES", StringType(), True), + StructField("READ_SINGLE_OPERATIONS", StringType(), True), + StructField("READ_SINGLE_AVERAGE", StringType(), True), + StructField("READ_SINGLE_SIGMA", StringType(), True), + StructField("READ_VECTOR_BYTES", StringType(), True), + StructField("READ_VECTOR_OPERATIONS", StringType(), True), + StructField("READ_VECTOR_AVERAGE", StringType(), True), + StructField("READ_VECTOR_SIGMA", StringType(), True), + StructField("READ_VECTOR_COUNT_AVERAGE", StringType(), True), + StructField("READ_VECTOR_COUNT_SIGMA", StringType(), True), + StructField("USER_DN", StringType(), True), + StructField("APP_INFO", StringType(), True), + StructField("START_TIME", StringType(), True), + StructField("END_TIME", StringType(), True), + StructField("START_DATE", LongType(), True), + StructField("END_DATE", LongType(), True), + StructField("INSERT_DATE", LongType(), True) + ]) + +def schema_jm(): + """Schema for JobMonitoring record + {"JobId":{"string":"1672451388"},"FileName":{"string":"//store/mc/PhaseIIFall16GS82/QCD_Flat_Pt-15to7000_TuneCUETP8M1_14TeV_pythia8/GEN-SIM/90X_upgrade2023_realistic_v1-v1/110000/92A9E11F-C0F1-E611-9A55-001E67E6F8AF.root"},"IsParentFile":{"string":"0"},"ProtocolUsed":{"string":"Remote"},"SuccessFlag":{"string":"1"},"FileType":{"string":"EDM"},"LumiRanges":{"string":"unknown"},"StrippedFiles":{"string":"0"},"BlockId":{"string":"602064"},"StrippedBlocks":{"string":"0"},"BlockName":{"string":"Dummy"},"InputCollection":{"string":"DoesNotApply"},"Application":{"string":"CMSSW"},"Type":{"string":"reprocessing"},"SubmissionTool":{"string":"wmagent"},"InputSE":null,"TargetCE":null,"SiteName":{"string":"T0_CH_CERN"},"SchedulerName":{"string":"PYCONDOR"},"JobMonitorId":{"string":"unknown"},"TaskJobId":{"string":"1566463230"},"SchedulerJobIdV2":{"string":"664eef36-f1c3-11e6-88b9-02163e0184a6-367_0"},"TaskId":{"string":"35076445"},"TaskMonitorId":{"string":"wmagent_pdmvserv_task_SMP-PhaseIIFall16GS82-00005__v1_T_170213_041344_640"},"JobExecExitCode":{"string":"0"},"JobExecExitTimeStamp":{"long":1488375506000},"StartedRunningTimeStamp":{"long":1488374686000},"FinishedTimeStamp":{"long":1488375506000},"WrapWC":{"string":"820"},"WrapCPU":{"string":"1694.3"},"ExeCPU":{"string":"0"},"UserId":{"string":"124370"},"GridName":{"string":"Alan Malta Rodrigues"}} + """ + return StructType([ + StructField("JobId", StringType(), True), + StructField("FileName", StringType(), True), + StructField("IsParentFile", StringType(), True), + StructField("ProtocolUsed", StringType(), True), + StructField("SuccessFlag", StringType(), True), + StructField("FileType", StringType(), True), + StructField("LumiRanges", StringType(), True), + StructField("StrippedFiles", StringType(), True), + StructField("BlockId", StringType(), True), + StructField("StrippedBlocks", StringType(), True), + StructField("BlockName", StringType(), True), + StructField("InputCollection", StringType(), True), + StructField("Application", StringType(), True), + StructField("ApplicationVersion", StringType(), True), + StructField("Type", StringType(), True), + StructField("GenericType", StringType(), True), + StructField("NewGenericType", StringType(), True), + StructField("NewType", StringType(), True), + StructField("SubmissionTool", StringType(), True), + StructField("InputSE", StringType(), True), + StructField("TargetCE", StringType(), True), + StructField("SiteName", StringType(), True), + StructField("SchedulerName", StringType(), True), + StructField("JobMonitorId", StringType(), True), + StructField("TaskJobId", StringType(), True), + StructField("SchedulerJobIdV2", StringType(), True), + StructField("TaskId", StringType(), True), + StructField("TaskMonitorId", StringType(), True), + StructField("NEventsPerJob", StringType(), True), + StructField("NTaskSteps", StringType(), True), + StructField("JobExecExitCode", StringType(), True), + StructField("JobExecExitTimeStamp", LongType(), True), + StructField("StartedRunningTimeStamp", LongType(), True), + StructField("FinishedTimeStamp", LongType(), True), + StructField("WrapWC", StringType(), True), + StructField("WrapCPU", StringType(), True), + StructField("ExeCPU", StringType(), True), + StructField("NCores", StringType(), True), + StructField("NEvProc", StringType(), True), + StructField("NEvReq", StringType(), True), + StructField("WNHostName", StringType(), True), + StructField("JobType", StringType(), True), + StructField("UserId", StringType(), True), + StructField("GridName", StringType(), True) + ]) + +def schema_asodb(): + """ + ASO table schema + Map for values of tm_transfer_state and tm_publication_state in + https://github.com/dmwm/CRABServer/blob/master/src/python/ServerUtilities.py#L61-L77 + + tm_id VARCHAR(60) NOT NULL, + tm_username VARCHAR(30) NOT NULL, + tm_taskname VARCHAR(255) NOT NULL, + tm_destination VARCHAR(100) NOT NULL, + tm_destination_lfn VARCHAR(1000) NOT NULL, + tm_source VARCHAR(100) NOT NULL, + tm_source_lfn VARCHAR(1000) NOT NULL, + tm_filesize NUMBER(20) NOT NULL, + tm_publish NUMBER(1) NOT NULL, + tm_jobid NUMBER(10) NOT NULL, + tm_job_retry_count NUMBER(5), + tm_type VARCHAR(20) NOT NULL, + tm_aso_worker VARCHAR(100), + tm_transfer_retry_count NUMBER(5) DEFAULT 0, + tm_transfer_max_retry_count NUMBER(5) DEFAULT 2, + tm_publication_retry_count NUMBER(5) DEFAULT 0, + tm_publication_max_retry_count NUMBER(5) DEFAULT 2, + tm_rest_host VARCHAR(50) NOT NULL, + tm_rest_uri VARCHAR(255) NOT NULL, + tm_transfer_state NUMBER(1) NOT NULL, + tm_publication_state NUMBER(1) NOT NULL, + tm_transfer_failure_reason VARCHAR(1000), + tm_publication_failure_reason VARCHAR(1000), + tm_fts_id VARCHAR(255), + tm_fts_instance VARCHAR(255), + tm_last_update NUMBER(11) NOT NULL, + tm_start_time NUMBER(11) NOT NULL, + tm_end_time NUMBER(11) + """ + return StructType([ + StructField("tm_id", StringType(), True), + StructField("tm_username", StringType(), True), + StructField("tm_taskname", StringType(), True), + StructField("tm_destination", StringType(), True), + StructField("tm_destination_lfn", StringType(), True), + StructField("tm_source", StringType(), True), + StructField("tm_source_lfn", StringType(), True), + StructField("tm_filesize", DoubleType(), True), + StructField("tm_publish", DoubleType(), True), + StructField("tm_jobid", IntegerType(), True), + StructField("tm_job_retry_count", IntegerType(), True), + StructField("tm_type", StringType(), True), + StructField("tm_aso_worker", StringType(), True), + StructField("tm_transfer_retry_count", IntegerType(), True), + StructField("tm_transfer_max_retry_count", IntegerType(), True), + StructField("tm_publication_retry_count", IntegerType(), True), + StructField("tm_publication_max_retry_count", IntegerType(), True), + StructField("tm_rest_host", StringType(), True), + StructField("tm_rest_uri", StringType(), True), + StructField("tm_transfer_state", IntegerType(), True), + StructField("tm_publication_state", IntegerType(), True), + StructField("tm_transfer_failure_reason", StringType(), True), + StructField("tm_publication_failure_reason", StringType(), True), + StructField("tm_fts_id", StringType(), True), + StructField("tm_fts_instance", StringType(), True), + StructField("tm_last_update", DoubleType(), True), + StructField("tm_start_time", DoubleType(), True), + StructField("tm_end_time", DoubleType(), True), + ]) + + +def aggregated_data_schema(): + """ + dn: string (nullable = true) + dataset_name: string (nullable = true) + site_name: string (nullable = true) + app: string (nullable = true) + uid: integer (nullable = true) + stream: string (nullable = true) + timestamp: integer (nullable = true) + nacc: integer (nullable = true) + distinct_users: integer (nullable = true) + site_tier: string (nullable = true) + cpu_time: double (nullable = true) + wc_time: double (nullable = true) + primary_name: string (nullable = true) + processing_name: string (nullable = true) + data_tier: string (nullable = true) + """ + + return StructType([ + StructField("dn", StringType(), True), + StructField("dataset_name", StringType(), True), + StructField("site_name", StringType(), True), + StructField("app", StringType(), True), + StructField("uid", LongType(), True), + StructField("stream", StringType(), True), + StructField("timestamp", LongType(), True), + StructField("nacc", IntegerType(), True), + StructField("distinct_users", IntegerType(), True), + StructField("site_tier", StringType(), True), + StructField("cpu_time", DoubleType(), True), + StructField("wc_time", DoubleType(), True), + StructField("primary_name", StringType(), True), + StructField("processing_name", StringType(), True), + StructField("data_tier", StringType(), True), + ]) + +def schema_empty_aaa(): + """ + src_experiment_site: string (nullable = true) + user_dn: string (nullable = true) + file_lfn: string (nullable = true) + """ + return StructType([ + StructField("src_experiment_site", StringType(), True), + StructField("user_dn", StringType(), True), + StructField("file_lfn", StringType(), True), + ]) + +def schema_empty_eos(): + """ + src_experiment_site: string (nullable = true) + """ + return StructType([ + StructField("file_lfn", StringType(), True), + StructField("user_dn", StringType(), True), + StructField("host", StringType(), True), + ]) diff --git a/consistency_check/spark/setup.sh b/consistency_check/spark/setup.sh new file mode 100644 index 0000000..a939216 --- /dev/null +++ b/consistency_check/spark/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source /cvmfs/sft.cern.ch/lcg/views/LCG_95a/x86_64-centos7-gcc7-opt/setup.sh +source /cvmfs/sft.cern.ch/lcg/etc/hadoop-confext/hadoop-swan-setconf.sh analytix +kinit From 93b13ce93e5363da914b5043bbda32fcdb913ba2 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Mon, 9 Dec 2019 16:15:27 -0600 Subject: [PATCH 022/192] Update README.md --- consistency_check/spark/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index 262037e..57be552 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -2,9 +2,9 @@ login to lxplus7.cern.ch -run setup.sh +**sh setup.sh** @@ -13,7 +13,6 @@ run setup.sh Example : python dbs_phedex_consistency.py --timestamp 2019-12-09 --dbs invalid --phedex present --output hdfs://analytix/user/pjasal/ - ## run file mismatch using Example : python filemismatch.py --timestamp 7 From 08da587452737dd90474c82f1809710a218267e1 Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Tue, 10 Dec 2019 21:00:43 +0100 Subject: [PATCH 023/192] minor change in arg parser --- .../spark/dbs_phedex_consistency.py | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 02883b0..ce8915f 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -3,15 +3,14 @@ import argparse import json import os -from datetime import datetime, timedelta from pyspark import SparkConf, SparkContext, StorageLevel from pyspark.sql import SparkSession, Column from pyspark.sql.functions import col -import pyspark.sql.functions as fna +import pyspark.sql.functions as fn import pyspark.sql.types as types from pyspark.sql import DataFrame - +import datetime # stolen from CMSSpark import schemas @@ -20,8 +19,6 @@ class OptionParser(): def __init__(self): "option Parser" self.parser = argparse.ArgumentParser(prog='consistency') - msg = "Optional required timestamp format yyyy-mm-dd , example : 2019-01-31 " - self.parser.add_argument("--timestamp",action="store",dest="phedex_time",default=None,help=msg) msg = "phedex dataset option: it takes 'missing' or 'present'" self.parser.add_argument("--phedex",action="store",dest="phedex_status",default=None,help=msg,required=True) msg = "DBS dataset status" @@ -29,14 +26,12 @@ def __init__(self): msg= "Output path in HDFS for result" self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) - - def run_consistency(args): + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") sc = SparkContext(conf=conf) spark = SparkSession(sc) print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) - phedex_time_stamp = valid_data(args.phedex_time) avroreader = spark.read.format("com.databricks.spark.avro") csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") @@ -44,6 +39,9 @@ def run_consistency(args): dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + yesturday = datetime.date.today() - datetime.timedelta(days=1) + phedex_time_stamp = yesturday.strftime("%Y-%m-%d") ## specifying date for phedex replicas + if args.phedex_status == 'present' and args.dbs_status == 'invalid': phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) @@ -65,7 +63,7 @@ def run_consistency(args): .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info .distinct()) working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby("input_campaign").agg((fn.count(fn.col("dataset_name").isNotNull())).show() + working_set.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() elif args.phedex_status == 'present' and args.dbs_status == 'deleted': phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) @@ -79,7 +77,7 @@ def run_consistency(args): .select('input_campaign','d_dataset') # you can select more columns for detail info .distinct()) working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby("input_campaign").agg((fn.count(fn.col("dataset_name").isNotNull())).show() + working_set.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() elif args.phedex_status == 'missing' and args.dbs_status == 'valid': # to be on the safe side : adding time difference for block replicas injected d = phedex_time_stamp - timedelta(days=30) # should add argumentparser option @@ -92,20 +90,11 @@ def run_consistency(args): .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) .filter(col('block_name').isNotNull() & col('node_id').isNull()) .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) - .select('block_name') + .select('input_campaign','block_name') .distinct()) working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby("input_campaign").agg((fn.count(fn.col("dataset_name").isNotNull())).show() + working_set.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() -def valid_data(args.phedex_time): - if args.phedex_time: - try : - return datetime.strptime(args.phedex_time,"%Y%m%d") - except ValueError: - msg = " Not a valid date : '{0}'.".format(args.phedex_time) - raise argparse.ArgumentTypeError(msg) - else: - return datetime.datetime.today().strftime("%Y%m%d") if __name__ == '__main__': optmgr = OptionParser() From dd5c5eca38350d65472d9665618f924eb86facb4 Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Tue, 10 Dec 2019 23:20:19 +0100 Subject: [PATCH 024/192] minor changes --- consistency_check/spark/filemismatch.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py index ea624c5..3124152 100644 --- a/consistency_check/spark/filemismatch.py +++ b/consistency_check/spark/filemismatch.py @@ -3,14 +3,15 @@ import argparse import json import os -from datetime import datetime, timedelta from pyspark import SparkConf, SparkContext, StorageLevel from pyspark.sql import SparkSession, Column from pyspark.sql.functions import col -import pyspark.sql.functions as fna +import pyspark.sql.functions as fn import pyspark.sql.types as types from pyspark.sql import DataFrame +import datetime +import time # stolen from CMSSpark @@ -21,7 +22,9 @@ def __init__(self): "option Parser" self.parser = argparse.ArgumentParser(prog='consistency') msg = "Files mismatch since last n days : default in past 7 days" - self.parser.add_argument("--timestamp",action="store",dest="timestamp",default=7,help=msg,type=int) + self.parser.add_argument("--days",action="store",dest="timestamp",default=7,help=msg,type=int) + msg= "Output path in HDFS for result" + self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) def fileMismatch(args): conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") @@ -34,21 +37,21 @@ def fileMismatch(args): dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") - current = datetime.datetime.utcnow().timestamp() + current = time.time() past_n_days = args.timestamp delta_t = current - past_n_days*60*60*24 - if args.phedex_status == 'present' and args.dbs_status == 'invalid': + if args.out_path: + out = args.out_path + "filemismatch" working_set = (dbs_files - .filter(col('f_is_file_valid')=='1') + .filter(col('f_is_file_valid')=='0') .filter(col('f_last_modification_date') >= delta_t) .join(dbs_datasets,col('f_dataset_id')==col('d_dataset_id')) .filter(col('d_dataset_access_type_id')=='1') - .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) - .select('input_campaign','d_dataset','d_last_modified_by','f_logical_file_name') # you can select more columns for detail info + .select('d_dataset','d_last_modified_by','f_logical_file_name') # you can select more columns for detail info .distinct()) working_set.select('f_logical_file_name').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby('d_dataset').agg((fn.count(fn.col("f_logical_file_name").isNotNull())).show() + working_set.groupby('d_dataset').agg((fn.count(fn.col("f_logical_file_name").isNotNull()))).show() if __name__ == '__main__': optmgr = OptionParser() args = optmgr.parser.parse_args() From f526a160d5be3f563cb5c931ae4c4277172c6318 Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Tue, 10 Dec 2019 23:25:38 +0100 Subject: [PATCH 025/192] modified variable name working_set to mismatch_df --- consistency_check/spark/filemismatch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py index 3124152..f6e5cce 100644 --- a/consistency_check/spark/filemismatch.py +++ b/consistency_check/spark/filemismatch.py @@ -43,15 +43,15 @@ def fileMismatch(args): if args.out_path: out = args.out_path + "filemismatch" - working_set = (dbs_files + mismatch_df = (dbs_files .filter(col('f_is_file_valid')=='0') .filter(col('f_last_modification_date') >= delta_t) .join(dbs_datasets,col('f_dataset_id')==col('d_dataset_id')) .filter(col('d_dataset_access_type_id')=='1') .select('d_dataset','d_last_modified_by','f_logical_file_name') # you can select more columns for detail info .distinct()) - working_set.select('f_logical_file_name').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby('d_dataset').agg((fn.count(fn.col("f_logical_file_name").isNotNull()))).show() + mismatch_df.select('f_logical_file_name').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(out) + mismatch_df.groupby('d_dataset').agg((fn.count(fn.col("f_logical_file_name").isNotNull()))).show() if __name__ == '__main__': optmgr = OptionParser() args = optmgr.parser.parse_args() From 06134f2acea95e9d973b785f744c4ea36591eefa Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Tue, 10 Dec 2019 23:30:08 +0100 Subject: [PATCH 026/192] variable name changed --- .../spark/dbs_phedex_consistency.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index ce8915f..f80dc90 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -12,6 +12,7 @@ from pyspark.sql import DataFrame import datetime + # stolen from CMSSpark import schemas @@ -54,7 +55,7 @@ def run_consistency(args): 81 : Deleted ''' out = args.out_path + "/invalid_dbs_present_phedex" - working_set = (dbs_datasets + invalid_dbs_present_phedex = (dbs_datasets .filter(col('d_dataset_access_type_id')=='2') .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) @@ -62,13 +63,13 @@ def run_consistency(args): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info .distinct()) - working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() + invalid_dbs_present_phedex.write.format("com.databricks.spark.csv").option("header", "true").save(out) + invalid_dbs_present_phedex.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() elif args.phedex_status == 'present' and args.dbs_status == 'deleted': phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/deleted_dbs_present_phedex" - working_set = (dbs_datasets + deleted_dbs_present_phedex = (dbs_datasets .filter(col('d_dataset_access_type_id')=='81') .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) @@ -76,15 +77,15 @@ def run_consistency(args): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset') # you can select more columns for detail info .distinct()) - working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() + deleted_dbs_present_phedex.write.format("com.databricks.spark.csv").option("header", "true").save(out) + deleted_dbs_present_phedex.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() elif args.phedex_status == 'missing' and args.dbs_status == 'valid': # to be on the safe side : adding time difference for block replicas injected - d = phedex_time_stamp - timedelta(days=30) # should add argumentparser option + d = phedex_time_stamp - datetime.timedelta(days=30) # should add argumentparser option phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(d) + "_*/part-m-00000" phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/valid_dbs_missing_phedex" - working_set = (dbs_datasets + valid_dbs_missing_phedex = (dbs_datasets .filter(col('d_dataset_access_type_id')=='1') .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) @@ -92,8 +93,8 @@ def run_consistency(args): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','block_name') .distinct()) - working_set.write.format("com.databricks.spark.csv").option("header", "true").save(out) - working_set.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() + valid_dbs_missing_phedex.write.format("com.databricks.spark.csv").option("header", "true").save(out) + valid_dbs_missing_phedex.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() if __name__ == '__main__': From 14aead018210571152967d17a51f117d64a10a94 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:51:10 -0600 Subject: [PATCH 027/192] Update README.md --- consistency_check/spark/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index 57be552..83ff252 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -1,4 +1,9 @@ -# Basic setup + +# Spark code run consistency between DBS and Phedex databases + + + +## Basic setup login to lxplus7.cern.ch @@ -13,6 +18,8 @@ login to lxplus7.cern.ch Example : python dbs_phedex_consistency.py --timestamp 2019-12-09 --dbs invalid --phedex present --output hdfs://analytix/user/pjasal/ -## run file mismatch using +## It returns the list of file which are invalidated by the unified for bad workflows Example : +--timestamp argument for checking the list for last n number of days + python filemismatch.py --timestamp 7 From 378f6c08250b10acf90234b0a7f5b98205e4d9fa Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:51:25 -0600 Subject: [PATCH 028/192] Update README.md --- consistency_check/spark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index 83ff252..94f86ca 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -1,5 +1,5 @@ -# Spark code run consistency between DBS and Phedex databases +# Spark code to run consistency between DBS and Phedex databases From 31c47d0bf040e3bfc5f413e08b76bd95759e4d2e Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:51:38 -0600 Subject: [PATCH 029/192] Update README.md --- consistency_check/spark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index 94f86ca..8da17b1 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -3,7 +3,7 @@ -## Basic setup +## Basic lxplus7 setup login to lxplus7.cern.ch From 863d423671db3feff472f0f631d30db67269371b Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:55:09 -0600 Subject: [PATCH 030/192] Update README.md --- consistency_check/spark/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index 8da17b1..dc915e7 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -15,7 +15,11 @@ login to lxplus7.cern.ch ## run dbs and phedex consistency check on spark: -Example : python dbs_phedex_consistency.py --timestamp 2019-12-09 --dbs invalid --phedex present --output hdfs://analytix/user/pjasal/ +argument --dbs takes values : "invalid" , "deleted" , "valid" +argument --phedex takes values : "missing" , "present" +argument --output specify the user area on analytix cluster + +Example : python dbs_phedex_consistency.py --dbs invalid --phedex present --output hdfs://analytix/user/username/ ## It returns the list of file which are invalidated by the unified for bad workflows From 9ec98bfd380992e7caffacdcfce5c34095a56309 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:56:35 -0600 Subject: [PATCH 031/192] Update README.md --- consistency_check/spark/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index dc915e7..9bdf834 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -15,15 +15,15 @@ login to lxplus7.cern.ch ## run dbs and phedex consistency check on spark: -argument --dbs takes values : "invalid" , "deleted" , "valid" -argument --phedex takes values : "missing" , "present" -argument --output specify the user area on analytix cluster +--dbs takes values : "invalid" , "deleted" , "valid" . +--phedex takes values : "missing" , "present" . +--output specify the user area on analytix cluster . Example : python dbs_phedex_consistency.py --dbs invalid --phedex present --output hdfs://analytix/user/username/ ## It returns the list of file which are invalidated by the unified for bad workflows Example : ---timestamp argument for checking the list for last n number of days +--timestamp argument for checking the list for last n number of days python filemismatch.py --timestamp 7 From 38baed0b876b2e339d77656c2ca3156735952d8c Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:56:57 -0600 Subject: [PATCH 032/192] Update README.md --- consistency_check/spark/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index 9bdf834..ffc28b3 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -15,9 +15,9 @@ login to lxplus7.cern.ch ## run dbs and phedex consistency check on spark: ---dbs takes values : "invalid" , "deleted" , "valid" . ---phedex takes values : "missing" , "present" . ---output specify the user area on analytix cluster . +--dbs takes values : "invalid" , "deleted" , "valid" \ +--phedex takes values : "missing" , "present" \ +--output specify the user area on analytix cluster \ Example : python dbs_phedex_consistency.py --dbs invalid --phedex present --output hdfs://analytix/user/username/ From 72e4010d9edb428da9be15ce71e9ca834fbd0ab7 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:57:15 -0600 Subject: [PATCH 033/192] Update README.md --- consistency_check/spark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index ffc28b3..fd5aea1 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -17,7 +17,7 @@ login to lxplus7.cern.ch --dbs takes values : "invalid" , "deleted" , "valid" \ --phedex takes values : "missing" , "present" \ ---output specify the user area on analytix cluster \ +--output specify the user area on analytix cluster Example : python dbs_phedex_consistency.py --dbs invalid --phedex present --output hdfs://analytix/user/username/ From a80950b647cad680bd3b8193d68600f5364d9164 Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Thu, 12 Dec 2019 21:35:29 +0100 Subject: [PATCH 034/192] checking only today replica --- consistency_check/spark/dbs_phedex_consistency.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index f80dc90..cf3b93e 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -40,8 +40,7 @@ def run_consistency(args): dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") - yesturday = datetime.date.today() - datetime.timedelta(days=1) - phedex_time_stamp = yesturday.strftime("%Y-%m-%d") ## specifying date for phedex replicas + phedex_time_stamp = datetime.date.today().strftime("%Y-%m-%d") ## specifying date for phedex replicas if args.phedex_status == 'present' and args.dbs_status == 'invalid': phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" From fc87cb8313c5b49287890d1146a179545f379d32 Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Mon, 16 Dec 2019 20:23:54 +0100 Subject: [PATCH 035/192] converted functions into a class --- .../spark/dbs_phedex_consistency.py | 67 ++++++++++--------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index cf3b93e..6b19fdf 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -20,31 +20,23 @@ class OptionParser(): def __init__(self): "option Parser" self.parser = argparse.ArgumentParser(prog='consistency') - msg = "phedex dataset option: it takes 'missing' or 'present'" - self.parser.add_argument("--phedex",action="store",dest="phedex_status",default=None,help=msg,required=True) - msg = "DBS dataset status" - self.parser.add_argument("--dbs",action="store",dest="dbs_status",default=None,help=msg,required=True) msg= "Output path in HDFS for result" self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) -def run_consistency(args): +class run_consistency(object): + def __init__(self, out): + self.out = out + self.avroreader = spark.read.format("com.databricks.spark.avro") + self.csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") + self.phedex_time_stamp = datetime.date.today().strftime("%Y-%m-%d") - conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") - sc = SparkContext(conf=conf) - spark = SparkSession(sc) - print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) - - avroreader = spark.read.format("com.databricks.spark.avro") - csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") - dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") - dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") - dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") - - phedex_time_stamp = datetime.date.today().strftime("%Y-%m-%d") ## specifying date for phedex replicas - - if args.phedex_status == 'present' and args.dbs_status == 'invalid': - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" - phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) + def invalid_dbs_present_phedex(self): + print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) + dbs_files = self.csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") + dbs_blocks = self.csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") + dbs_datasets = self.csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" + phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) ''' for reference dbs d_dataset_access_type_id: 1 : valid @@ -53,7 +45,6 @@ def run_consistency(args): 41 : Production 81 : Deleted ''' - out = args.out_path + "/invalid_dbs_present_phedex" invalid_dbs_present_phedex = (dbs_datasets .filter(col('d_dataset_access_type_id')=='2') .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) @@ -62,10 +53,14 @@ def run_consistency(args): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info .distinct()) - invalid_dbs_present_phedex.write.format("com.databricks.spark.csv").option("header", "true").save(out) invalid_dbs_present_phedex.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() - elif args.phedex_status == 'present' and args.dbs_status == 'deleted': - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(phedex_time_stamp) + "_*/part-m-00000" + return invalid_dbs_present_phedex.select("d_dataset") + + def deleted_dbs_present_phedex(self): + dbs_files = self.csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") + dbs_blocks = self.csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") + dbs_datasets = self.csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/deleted_dbs_present_phedex" deleted_dbs_present_phedex = (dbs_datasets @@ -76,13 +71,16 @@ def run_consistency(args): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset') # you can select more columns for detail info .distinct()) - deleted_dbs_present_phedex.write.format("com.databricks.spark.csv").option("header", "true").save(out) deleted_dbs_present_phedex.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() - elif args.phedex_status == 'missing' and args.dbs_status == 'valid': + return deleted_dbs_present_phedex.select("d_dataset") + + def valid_dbs_missing_phedex(self): # to be on the safe side : adding time difference for block replicas injected - d = phedex_time_stamp - datetime.timedelta(days=30) # should add argumentparser option - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(d) + "_*/part-m-00000" - phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) + dbs_files = self.csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") + dbs_blocks = self.csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") + dbs_datasets = self.csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" + phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/valid_dbs_missing_phedex" valid_dbs_missing_phedex = (dbs_datasets .filter(col('d_dataset_access_type_id')=='1') @@ -92,11 +90,16 @@ def run_consistency(args): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','block_name') .distinct()) - valid_dbs_missing_phedex.write.format("com.databricks.spark.csv").option("header", "true").save(out) valid_dbs_missing_phedex.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() + return valid_dbs_missing_phedex.select("block_name") if __name__ == '__main__': optmgr = OptionParser() args = optmgr.parser.parse_args() - run_consistency(args) + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + spark = SparkSession(sc) + con = run_consistency(args.out_path) + df = con.valid_dbs_missing_phedex() + df.show() From 6c885f4f24f394d5f30977e2c4572ef5f2871449 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Mon, 16 Dec 2019 13:37:25 -0600 Subject: [PATCH 036/192] Update dbs_phedex_consistency.py --- consistency_check/spark/dbs_phedex_consistency.py | 1 - 1 file changed, 1 deletion(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 6b19fdf..88e1c47 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -102,4 +102,3 @@ def valid_dbs_missing_phedex(self): spark = SparkSession(sc) con = run_consistency(args.out_path) df = con.valid_dbs_missing_phedex() - df.show() From 4db3047c6e0f3793a1e500ce0bef8c96c932c756 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Thu, 2 Jan 2020 14:16:51 -0600 Subject: [PATCH 037/192] Update README.md --- consistency_check/spark/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index fd5aea1..ce638f1 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -15,11 +15,6 @@ login to lxplus7.cern.ch ## run dbs and phedex consistency check on spark: ---dbs takes values : "invalid" , "deleted" , "valid" \ ---phedex takes values : "missing" , "present" \ ---output specify the user area on analytix cluster - -Example : python dbs_phedex_consistency.py --dbs invalid --phedex present --output hdfs://analytix/user/username/ ## It returns the list of file which are invalidated by the unified for bad workflows From 425f4d28cc742137be6e4b7d81a839e777ade4ed Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Thu, 2 Jan 2020 14:27:34 -0600 Subject: [PATCH 038/192] Update dbs_phedex_consistency.py --- consistency_check/spark/dbs_phedex_consistency.py | 1 - 1 file changed, 1 deletion(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 88e1c47..d3c6451 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -75,7 +75,6 @@ def deleted_dbs_present_phedex(self): return deleted_dbs_present_phedex.select("d_dataset") def valid_dbs_missing_phedex(self): - # to be on the safe side : adding time difference for block replicas injected dbs_files = self.csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") dbs_blocks = self.csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") dbs_datasets = self.csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") From f7bb2d7bd67568d9e1fa3a4b98a7e47e1de9207b Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 8 Jan 2020 17:01:37 -0600 Subject: [PATCH 039/192] Update filemismatch.py --- consistency_check/spark/filemismatch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py index f6e5cce..b7b4c63 100644 --- a/consistency_check/spark/filemismatch.py +++ b/consistency_check/spark/filemismatch.py @@ -34,8 +34,8 @@ def fileMismatch(args): avroreader = spark.read.format("com.databricks.spark.avro") csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") - dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") - dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") + dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") current = time.time() past_n_days = args.timestamp @@ -45,7 +45,7 @@ def fileMismatch(args): out = args.out_path + "filemismatch" mismatch_df = (dbs_files .filter(col('f_is_file_valid')=='0') - .filter(col('f_last_modification_date') >= delta_t) + .filter(col('f_last_modification_date') > delta_t) .join(dbs_datasets,col('f_dataset_id')==col('d_dataset_id')) .filter(col('d_dataset_access_type_id')=='1') .select('d_dataset','d_last_modified_by','f_logical_file_name') # you can select more columns for detail info From 635c662ac8fadf9d2ba679d128efc05db4c7f6df Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Thu, 9 Jan 2020 20:31:33 +0100 Subject: [PATCH 040/192] minor changes --- consistency_check/spark/filemismatch.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py index f6e5cce..a5949d8 100644 --- a/consistency_check/spark/filemismatch.py +++ b/consistency_check/spark/filemismatch.py @@ -22,7 +22,7 @@ def __init__(self): "option Parser" self.parser = argparse.ArgumentParser(prog='consistency') msg = "Files mismatch since last n days : default in past 7 days" - self.parser.add_argument("--days",action="store",dest="timestamp",default=7,help=msg,type=int) + self.parser.add_argument("--past_days",action="store",dest="days",default=7,help=msg,type=int) msg= "Output path in HDFS for result" self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) @@ -34,24 +34,24 @@ def fileMismatch(args): avroreader = spark.read.format("com.databricks.spark.avro") csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") - dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") - dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") + dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") current = time.time() - past_n_days = args.timestamp + past_n_days = args.days delta_t = current - past_n_days*60*60*24 if args.out_path: - out = args.out_path + "filemismatch" mismatch_df = (dbs_files .filter(col('f_is_file_valid')=='0') .filter(col('f_last_modification_date') >= delta_t) .join(dbs_datasets,col('f_dataset_id')==col('d_dataset_id')) .filter(col('d_dataset_access_type_id')=='1') - .select('d_dataset','d_last_modified_by','f_logical_file_name') # you can select more columns for detail info + .filter(col('f_logical_file_name').isNotNull()) + .select('d_dataset','f_last_modified_by','f_logical_file_name') .distinct()) - mismatch_df.select('f_logical_file_name').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(out) - mismatch_df.groupby('d_dataset').agg((fn.count(fn.col("f_logical_file_name").isNotNull()))).show() + mismatch_df.select('f_logical_file_name','f_last_modified_by').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(args.out_path) + mismatch_df.groupby('d_dataset').agg(fn.count(fn.col("f_logical_file_name")).alias('extra_lfn_phedex')).show() if __name__ == '__main__': optmgr = OptionParser() args = optmgr.parser.parse_args() From 890a05dff93737ab2f2a5cfeca94b07a8fdf866b Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Tue, 28 Jan 2020 17:49:08 -0600 Subject: [PATCH 041/192] minor changes --- .../spark/dbs_phedex_consistency.py | 45 +++++-------- scripts/checkAdler32_forFilesAtSites copy.py | 64 +++++++++++++++++++ 2 files changed, 80 insertions(+), 29 deletions(-) create mode 100644 scripts/checkAdler32_forFilesAtSites copy.py diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index d3c6451..1d08373 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -11,8 +11,6 @@ import pyspark.sql.types as types from pyspark.sql import DataFrame import datetime - - # stolen from CMSSpark import schemas @@ -26,17 +24,15 @@ def __init__(self): class run_consistency(object): def __init__(self, out): self.out = out - self.avroreader = spark.read.format("com.databricks.spark.avro") - self.csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") - self.phedex_time_stamp = datetime.date.today().strftime("%Y-%m-%d") + avroreader = spark.read.format("com.databricks.spark.avro") + csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(datetime.date.today().strftime("%Y-%m-%d")) + "_*/part-m-00000" + self.phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) + self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") + self.dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/BLOCKS/part-m-00000") + self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000" def invalid_dbs_present_phedex(self): - print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) - dbs_files = self.csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") - dbs_blocks = self.csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") - dbs_datasets = self.csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" - phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) ''' for reference dbs d_dataset_access_type_id: 1 : valid @@ -45,10 +41,10 @@ def invalid_dbs_present_phedex(self): 41 : Production 81 : Deleted ''' - invalid_dbs_present_phedex = (dbs_datasets + invalid_dbs_present_phedex = (self.dbs_datasets .filter(col('d_dataset_access_type_id')=='2') - .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) - .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) .filter(col('dataset_name').isNotNull()) .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info @@ -57,16 +53,11 @@ def invalid_dbs_present_phedex(self): return invalid_dbs_present_phedex.select("d_dataset") def deleted_dbs_present_phedex(self): - dbs_files = self.csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") - dbs_blocks = self.csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") - dbs_datasets = self.csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" - phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/deleted_dbs_present_phedex" - deleted_dbs_present_phedex = (dbs_datasets + deleted_dbs_present_phedex = (self.dbs_datasets .filter(col('d_dataset_access_type_id')=='81') - .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) - .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) .filter(col('dataset_name').isNotNull()) .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset') # you can select more columns for detail info @@ -75,16 +66,13 @@ def deleted_dbs_present_phedex(self): return deleted_dbs_present_phedex.select("d_dataset") def valid_dbs_missing_phedex(self): - dbs_files = self.csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") - dbs_blocks = self.csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/BLOCKS/part-m-00000") - dbs_datasets = self.csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/valid_dbs_missing_phedex" - valid_dbs_missing_phedex = (dbs_datasets + valid_dbs_missing_phedex = (self.dbs_datasets .filter(col('d_dataset_access_type_id')=='1') - .join(dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) - .join(phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) .filter(col('block_name').isNotNull() & col('node_id').isNull()) .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','block_name') @@ -100,4 +88,3 @@ def valid_dbs_missing_phedex(self): sc = SparkContext(conf=conf) spark = SparkSession(sc) con = run_consistency(args.out_path) - df = con.valid_dbs_missing_phedex() diff --git a/scripts/checkAdler32_forFilesAtSites copy.py b/scripts/checkAdler32_forFilesAtSites copy.py new file mode 100644 index 0000000..e1ed21c --- /dev/null +++ b/scripts/checkAdler32_forFilesAtSites copy.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +import os +import sys +import subprocess +import urllib +import json + +## This tool will get all replicas register for a particular LFN and will return the CMS sites + PFN + checksum (of the corrupt file). +## USAGE: python checkAdler32ForFilesAtSites.py + +def init(): + f = str(sys.argv[1]) + get_cksum_from_catalog(f) + get_file_replicas(f) + get_cksum_from_replica(f) + +def get_cksum_from_catalog(lfn): + url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/data?file=' + lfn + result = json.load(urllib.urlopen(url)) + for p in result['phedex']['dbs']: + for d in p['dataset']: + for b in d['block']: + for f in b['file']: + adler32.append(str(f['checksum']).split(',')[0].split(':')[1]) + +def get_file_replicas(lfn): + url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/filereplicas?lfn=' + lfn + result = json.load(urllib.urlopen(url)) + for b in result['phedex']['block']: + for f in b['file']: + for r in f['replica']: + sites.append(r['node']) + +def get_cksum_from_replica(lfn): + if len(adler32[0]) > 0: + print "-----------------------" + print "FILE: " + lfn + print "TMDB: " + adler32[0] + print "Replicas: (" + str(len(sites)) + ") " + str(sites).replace("u'","").replace("'","").replace("[","").replace("]","") + print "-----------------------" + if len(sites) == 0: + print "no corrupt files found" + else: + cf = 0 + for s in range(len(sites)): + url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn?protocol=srmv2&lfn=' + str(lfn) + '&node=' + str(sites[s]) + result = json.load(urllib.urlopen(url)) + for m in result['phedex']['mapping']: + pfn.append(m['pfn']) + cmd = "gfal-sum " + pfn[s] + " adler32" + result = subprocess.check_output(cmd, shell=True) + cksm = str(result).split('.root ')[1] + if str(adler32[0]).strip() != str(cksm).strip(): + print "site: " + sites[s] + " - " + result + cf = cf + 1 + print str(cf) + " site(s) with corrupt files found" + else: + print "no checksum info in TMDB for: " + lfn + +sites = [] +adler32 = [] +pfn = [] +corrupted = [] +init() From d3615a99bd2956eefdc6d6f09dc21f59cc6b726e Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 28 Jan 2020 18:27:28 -0600 Subject: [PATCH 042/192] Update dbs_phedex_consistency.py --- consistency_check/spark/dbs_phedex_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 1d08373..8304f3b 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -30,7 +30,7 @@ def __init__(self, out): self.phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") self.dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/BLOCKS/part-m-00000") - self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000" + self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") def invalid_dbs_present_phedex(self): ''' From 3647bc76810e966447443729faf4c0191ea86203 Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Wed, 29 Jan 2020 03:32:42 +0100 Subject: [PATCH 043/192] minor changes --- .../spark/dbs_phedex_inconsistency.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 consistency_check/spark/dbs_phedex_inconsistency.py diff --git a/consistency_check/spark/dbs_phedex_inconsistency.py b/consistency_check/spark/dbs_phedex_inconsistency.py new file mode 100644 index 0000000..939fef8 --- /dev/null +++ b/consistency_check/spark/dbs_phedex_inconsistency.py @@ -0,0 +1,91 @@ +#!/usr/bin/env spark-submit +from __future__ import print_function +import argparse +import json +import os + +from pyspark import SparkConf, SparkContext, StorageLevel +from pyspark.sql import SparkSession, Column +from pyspark.sql.functions import col +import pyspark.sql.functions as fn +import pyspark.sql.types as types +from pyspark.sql import DataFrame +from datetime import date , timedelta +# stolen from CMSSpark +import schemas + +class OptionParser(): + def __init__(self): + "option Parser" + self.parser = argparse.ArgumentParser(prog='consistency') + msg= "Output path in HDFS for result" + self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) + +class run_consistency(object): + def __init__(self, out): + self.out = out + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + self.spark = SparkSession(sc) + avroreader = self.spark.read.format("com.databricks.spark.avro") + csvreader = self.spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str((date.today() - timedelta(days=2)).strftime("%Y-%m-%d")) + "_*/part-m-00000" + self.phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) + self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") + self.dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/BLOCKS/part-m-00000") + self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") + + def invalid_dbs_present_phedex(self): + ''' + for reference dbs d_dataset_access_type_id: + 1 : valid + 2 : invalid + 42 : Deprecated + 41 : Production + 81 : Deleted + ''' + invalid_dbs_present_phedex = (self.dbs_datasets + .filter(col('d_dataset_access_type_id')=='2') + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('dataset_name').isNotNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info + .distinct()) + invalid_dbs_present_phedex.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() + return invalid_dbs_present_phedex.select("d_dataset") + + def deleted_dbs_present_phedex(self): + out = args.out_path + "/deleted_dbs_present_phedex" + deleted_dbs_present_phedex = (self.dbs_datasets + .filter(col('d_dataset_access_type_id')=='81') + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('dataset_name').isNotNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','d_dataset') # you can select more columns for detail info + .distinct()) + deleted_dbs_present_phedex.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() + return deleted_dbs_present_phedex.select("d_dataset") + + def valid_dbs_missing_phedex(self): + out = args.out_path + "/valid_dbs_missing_phedex" + valid_dbs_missing_phedex = (self.dbs_datasets + .filter(col('d_dataset_access_type_id')=='1') + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('block_name').isNotNull() & col('node_id').isNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','block_name') + .distinct()) + valid_dbs_missing_phedex.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() + return valid_dbs_missing_phedex.select("block_name") + + + +if __name__ == '__main__': + optmgr = OptionParser() + args = optmgr.parser.parse_args() + con = run_consistency(args.out_path) + invalid_dbs_present_phedex = con.deleted_dbs_present_phedex() + invalid_dbs_present_phedex.show(20,False) From 7a704a8c8a571162a9122db1a31efe1755bf95db Mon Sep 17 00:00:00 2001 From: Pradeep Jasal Date: Wed, 29 Jan 2020 03:35:07 +0100 Subject: [PATCH 044/192] minor changes --- .../spark/dbs_phedex_consistency.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 8304f3b..939fef8 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -10,7 +10,7 @@ import pyspark.sql.functions as fn import pyspark.sql.types as types from pyspark.sql import DataFrame -import datetime +from datetime import date , timedelta # stolen from CMSSpark import schemas @@ -24,10 +24,13 @@ def __init__(self): class run_consistency(object): def __init__(self, out): self.out = out - avroreader = spark.read.format("com.databricks.spark.avro") - csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(datetime.date.today().strftime("%Y-%m-%d")) + "_*/part-m-00000" - self.phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + self.spark = SparkSession(sc) + avroreader = self.spark.read.format("com.databricks.spark.avro") + csvreader = self.spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str((date.today() - timedelta(days=2)).strftime("%Y-%m-%d")) + "_*/part-m-00000" + self.phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") self.dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/BLOCKS/part-m-00000") self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") @@ -66,8 +69,6 @@ def deleted_dbs_present_phedex(self): return deleted_dbs_present_phedex.select("d_dataset") def valid_dbs_missing_phedex(self): - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" - phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/valid_dbs_missing_phedex" valid_dbs_missing_phedex = (self.dbs_datasets .filter(col('d_dataset_access_type_id')=='1') @@ -81,10 +82,10 @@ def valid_dbs_missing_phedex(self): return valid_dbs_missing_phedex.select("block_name") + if __name__ == '__main__': optmgr = OptionParser() args = optmgr.parser.parse_args() - conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") - sc = SparkContext(conf=conf) - spark = SparkSession(sc) con = run_consistency(args.out_path) + invalid_dbs_present_phedex = con.deleted_dbs_present_phedex() + invalid_dbs_present_phedex.show(20,False) From 92dba751e37bde15dada49e47d4cbae84a455cd8 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Tue, 28 Jan 2020 20:35:55 -0600 Subject: [PATCH 045/192] Delete dbs_phedex_inconsistency.py --- .../spark/dbs_phedex_inconsistency.py | 91 ------------------- 1 file changed, 91 deletions(-) delete mode 100644 consistency_check/spark/dbs_phedex_inconsistency.py diff --git a/consistency_check/spark/dbs_phedex_inconsistency.py b/consistency_check/spark/dbs_phedex_inconsistency.py deleted file mode 100644 index 939fef8..0000000 --- a/consistency_check/spark/dbs_phedex_inconsistency.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env spark-submit -from __future__ import print_function -import argparse -import json -import os - -from pyspark import SparkConf, SparkContext, StorageLevel -from pyspark.sql import SparkSession, Column -from pyspark.sql.functions import col -import pyspark.sql.functions as fn -import pyspark.sql.types as types -from pyspark.sql import DataFrame -from datetime import date , timedelta -# stolen from CMSSpark -import schemas - -class OptionParser(): - def __init__(self): - "option Parser" - self.parser = argparse.ArgumentParser(prog='consistency') - msg= "Output path in HDFS for result" - self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) - -class run_consistency(object): - def __init__(self, out): - self.out = out - conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") - sc = SparkContext(conf=conf) - self.spark = SparkSession(sc) - avroreader = self.spark.read.format("com.databricks.spark.avro") - csvreader = self.spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str((date.today() - timedelta(days=2)).strftime("%Y-%m-%d")) + "_*/part-m-00000" - self.phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) - self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") - self.dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/BLOCKS/part-m-00000") - self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") - - def invalid_dbs_present_phedex(self): - ''' - for reference dbs d_dataset_access_type_id: - 1 : valid - 2 : invalid - 42 : Deprecated - 41 : Production - 81 : Deleted - ''' - invalid_dbs_present_phedex = (self.dbs_datasets - .filter(col('d_dataset_access_type_id')=='2') - .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) - .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) - .filter(col('dataset_name').isNotNull()) - .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) - .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info - .distinct()) - invalid_dbs_present_phedex.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() - return invalid_dbs_present_phedex.select("d_dataset") - - def deleted_dbs_present_phedex(self): - out = args.out_path + "/deleted_dbs_present_phedex" - deleted_dbs_present_phedex = (self.dbs_datasets - .filter(col('d_dataset_access_type_id')=='81') - .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) - .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) - .filter(col('dataset_name').isNotNull()) - .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) - .select('input_campaign','d_dataset') # you can select more columns for detail info - .distinct()) - deleted_dbs_present_phedex.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() - return deleted_dbs_present_phedex.select("d_dataset") - - def valid_dbs_missing_phedex(self): - out = args.out_path + "/valid_dbs_missing_phedex" - valid_dbs_missing_phedex = (self.dbs_datasets - .filter(col('d_dataset_access_type_id')=='1') - .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) - .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) - .filter(col('block_name').isNotNull() & col('node_id').isNull()) - .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) - .select('input_campaign','block_name') - .distinct()) - valid_dbs_missing_phedex.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() - return valid_dbs_missing_phedex.select("block_name") - - - -if __name__ == '__main__': - optmgr = OptionParser() - args = optmgr.parser.parse_args() - con = run_consistency(args.out_path) - invalid_dbs_present_phedex = con.deleted_dbs_present_phedex() - invalid_dbs_present_phedex.show(20,False) From cf2a6d2feb62a19048370c104b23c484ae2fe5d1 Mon Sep 17 00:00:00 2001 From: pradeepjasal Date: Wed, 29 Jan 2020 10:07:54 -0600 Subject: [PATCH 046/192] minor changes --- .../spark/dbs_phedex_consistency.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 1d08373..07beb09 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -10,7 +10,7 @@ import pyspark.sql.functions as fn import pyspark.sql.types as types from pyspark.sql import DataFrame -import datetime +from datetime import date , timedelta # stolen from CMSSpark import schemas @@ -24,13 +24,17 @@ def __init__(self): class run_consistency(object): def __init__(self, out): self.out = out - avroreader = spark.read.format("com.databricks.spark.avro") - csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(datetime.date.today().strftime("%Y-%m-%d")) + "_*/part-m-00000" - self.phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + self.spark = SparkSession(sc) + avroreader = self.spark.read.format("com.databricks.spark.avro") + csvreader = self.spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") + ## check if path exist or not on hdfs area and you can directly assign it + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str((date.today() - timedelta(days=2)).strftime("%Y-%m-%d")) + "_*/part-m-00000" + self.phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") self.dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/BLOCKS/part-m-00000") - self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000" + self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") def invalid_dbs_present_phedex(self): ''' @@ -66,8 +70,6 @@ def deleted_dbs_present_phedex(self): return deleted_dbs_present_phedex.select("d_dataset") def valid_dbs_missing_phedex(self): - phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str(self.phedex_time_stamp) + "_*/part-m-00000" - phedex_block_replicas = self.csvreader.schema(schemas.schema_phedex()).load(phedex_path) out = args.out_path + "/valid_dbs_missing_phedex" valid_dbs_missing_phedex = (self.dbs_datasets .filter(col('d_dataset_access_type_id')=='1') @@ -81,10 +83,9 @@ def valid_dbs_missing_phedex(self): return valid_dbs_missing_phedex.select("block_name") + if __name__ == '__main__': optmgr = OptionParser() args = optmgr.parser.parse_args() - conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") - sc = SparkContext(conf=conf) - spark = SparkSession(sc) con = run_consistency(args.out_path) + invalid_dbs_present_phedex = con.deleted_dbs_present_phedex() From 86e367806cbf797425e0d61ba932294a7a498b20 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 29 Jan 2020 10:13:33 -0600 Subject: [PATCH 047/192] Update dbs_phedex_consistency.py --- consistency_check/spark/dbs_phedex_consistency.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 8fc26ef..1b94838 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -29,10 +29,7 @@ def __init__(self, out): self.spark = SparkSession(sc) avroreader = self.spark.read.format("com.databricks.spark.avro") csvreader = self.spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") -<<<<<<< HEAD - ## check if path exist or not on hdfs area and you can directly assign it -======= ->>>>>>> 92dba751e37bde15dada49e47d4cbae84a455cd8 + ## check if phedex_path exist or not on hdfs area and you can directly assign it phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str((date.today() - timedelta(days=2)).strftime("%Y-%m-%d")) + "_*/part-m-00000" self.phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") @@ -92,7 +89,3 @@ def valid_dbs_missing_phedex(self): args = optmgr.parser.parse_args() con = run_consistency(args.out_path) invalid_dbs_present_phedex = con.deleted_dbs_present_phedex() -<<<<<<< HEAD -======= - invalid_dbs_present_phedex.show(20,False) ->>>>>>> 92dba751e37bde15dada49e47d4cbae84a455cd8 From af7282f4675bfda9b613199e8bd142d1259d746a Mon Sep 17 00:00:00 2001 From: pradeepjasal Date: Wed, 29 Jan 2020 10:27:48 -0600 Subject: [PATCH 048/192] added some description of functions --- .../spark/dbs_phedex_consistency.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 1b94838..9ba8a1a 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -38,6 +38,11 @@ def __init__(self, out): def invalid_dbs_present_phedex(self): ''' + + Returns a dataframe with datasets which have "INVALID" status in DBS and are "PRESENT" in phedex + + :func: run_consistency.deleted_dbs_present_phedex() + for reference dbs d_dataset_access_type_id: 1 : valid 2 : invalid @@ -53,10 +58,17 @@ def invalid_dbs_present_phedex(self): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info .distinct()) + invalid_dbs_present_phedex.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() return invalid_dbs_present_phedex.select("d_dataset") def deleted_dbs_present_phedex(self): + ''' + Returns a dataframe with datasets which have "DELETED" status in DBS and "PRESENT" status in phedex + + :func: run_consistency.deleted_dbs_present_phedex() + + ''' out = args.out_path + "/deleted_dbs_present_phedex" deleted_dbs_present_phedex = (self.dbs_datasets .filter(col('d_dataset_access_type_id')=='81') @@ -66,10 +78,17 @@ def deleted_dbs_present_phedex(self): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','d_dataset') # you can select more columns for detail info .distinct()) + deleted_dbs_present_phedex.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() return deleted_dbs_present_phedex.select("d_dataset") def valid_dbs_missing_phedex(self): + ''' + Returns a dataframe with block name which have "VALID" status in DBS and are "MISSING" in phedex + + :func: run_consistency.deleted_dbs_present_phedex() + + ''' out = args.out_path + "/valid_dbs_missing_phedex" valid_dbs_missing_phedex = (self.dbs_datasets .filter(col('d_dataset_access_type_id')=='1') @@ -79,6 +98,8 @@ def valid_dbs_missing_phedex(self): .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) .select('input_campaign','block_name') .distinct()) + + print("Please check the below list of datasets with phedex api calls , Its possible you encounter discrepency due to the time difference of dataset injection in both databases and snapshot stored on HDFS") valid_dbs_missing_phedex.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() return valid_dbs_missing_phedex.select("block_name") From ca282cd99c7a58ef6197044c84fca7336d1ef9b1 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:41:09 -0600 Subject: [PATCH 049/192] Delete aaa_federation.py --- AAAOps/AAA_Subscription/aaa_federation.py | 77 ----------------------- 1 file changed, 77 deletions(-) delete mode 100755 AAAOps/AAA_Subscription/aaa_federation.py diff --git a/AAAOps/AAA_Subscription/aaa_federation.py b/AAAOps/AAA_Subscription/aaa_federation.py deleted file mode 100755 index be408eb..0000000 --- a/AAAOps/AAA_Subscription/aaa_federation.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import time -import subprocess -import urllib -import json -from datetime import datetime -import requests -import collections -import stomp -import argparse -from CMSMonitoring.StompAMQ import StompAMQ -import uuid -import hashlib -import stomp -from uuid import uuid4 -import time - - -class OptionParser(): - def __init__(self): - "User based option parser" - self.parser = argparse.ArgumentParser() - msg = "Send aaa subscription json file via StompAMQ to a broker, provide broker credentials in JSON file" - self.parser.add_argument("--amq", action="store", - dest="amq", default="", help=msg, required=True) - msg = " provide data json file" - self.parser.add_argument("--data", action="store", - dest="subscription_data", default="", help=msg, required=True) - - -def credentials(fname=None): - "Read credentials from MONIT_BROKER environment" - if not fname: - fname = os.environ.get('MONIT_BROKER', '') - if not os.path.isfile(fname): - raise Exception( - "Unable to locate MONIT credentials, please setup MONIT_BROKER") - return {} - with open(fname, 'r') as istream: - data = json.load(istream) - return data - - -if __name__ == "__main__": - optmgr = OptionParser() - opts = optmgr.parser.parse_args() - if opts.subscription_data: - try: - with open(opts.subscription_data, 'r') as f: - payload = json.load(f) - except IOError: - print("File not accessible") - - if opts.amq: - creds = credentials(opts.amq) - username = creds['username'] - password = creds['password'] - topic = creds['topic'] - host, port = creds['host_and_ports'].split(':') - port = int(port) - ckey = os.path.join("/root/.globus/", 'userkey.pem') - cert = os.path.join("/root/.globus/", 'usercert.pem') - producer = creds['producer'] - topic = creds['topic'] - if creds and StompAMQ: - amq = StompAMQ(username, password, producer, topic, key=ckey, - cert=cert, validation_schema=None, host_and_ports=[(host, port)]) - data = [] - for doc in payload: - hid = int(uuid4()) - notification, _, _ = amq.make_notification(doc, hid) - data.append(notification) - results = amq.send(data) - print("results", results) From 1edd736f8baa82b2543c66f7ca9d0713ca992412 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:41:21 -0600 Subject: [PATCH 050/192] Delete aaa_federation.sh --- AAAOps/AAA_Subscription/aaa_federation.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 AAAOps/AAA_Subscription/aaa_federation.sh diff --git a/AAAOps/AAA_Subscription/aaa_federation.sh b/AAAOps/AAA_Subscription/aaa_federation.sh deleted file mode 100755 index 261d01f..0000000 --- a/AAAOps/AAA_Subscription/aaa_federation.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - - -set -x -export PYTHONPATH=$PYTHONPATH:/root/pradeep/send_aaa_matrics/CMSMonitoring/src/python/ -python $PWD/test.py --amq /root/pradeep/send_aaa_matrics/credentials.json --data $PWD/aaa_schema.json From 2a8bbf94e1dde85c218e757cd42a73cc8cb7f057 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:41:28 -0600 Subject: [PATCH 051/192] Delete aaa_schema.json --- AAAOps/AAA_Subscription/aaa_schema.json | 116 ------------------------ 1 file changed, 116 deletions(-) delete mode 100644 AAAOps/AAA_Subscription/aaa_schema.json diff --git a/AAAOps/AAA_Subscription/aaa_schema.json b/AAAOps/AAA_Subscription/aaa_schema.json deleted file mode 100644 index f4ea273..0000000 --- a/AAAOps/AAA_Subscription/aaa_schema.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "T0_CH_CERN": "nowhere", - "T0_CH_CSCS_HPC": "nowhere", - "T1_DE_KIT": "prod", - "T1_ES_PIC": "prod", - "T1_FR_CCIN2P3": "prod", - "T1_IT_CNAF": "prod", - "T1_RU_JINR": "prod", - "T1_UK_RAL": "prod", - "T1_US_FNAL": "prod", - "T2_AT_Vienna": "prod", - "T2_BE_IIHE": "prod", - "T2_BE_UCL": "prod", - "T2_BR_SPRACE": "prod", - "T2_BR_UERJ": "prod", - "T2_CH_CERN": "prod", - "T2_CH_CERN_HLT": "nowhere", - "T2_CH_CSCS": "prod", - "T2_CN_Beijing": "prod", - "T2_DE_DESY": "prod", - "T2_DE_RWTH": "prod", - "T2_EE_Estonia": "prod", - "T2_ES_CIEMAT": "prod", - "T2_ES_IFCA": "prod", - "T2_FI_HIP": "prod", - "T2_FR_CCIN2P3": "prod", - "T2_FR_GRIF_IRFU": "prod", - "T2_FR_GRIF_LLR": "prod", - "T2_FR_IPHC": "prod", - "T2_GR_Ioannina": "prod", - "T2_HU_Budapest": "prod", - "T2_IN_TIFR": "prod", - "T2_IT_Bari": "prod", - "T2_IT_Legnaro": "prod", - "T2_IT_Pisa": "prod", - "T2_IT_Rome": "prod", - "T2_KR_KISTI": "prod", - "T2_PK_NCP": "trans", - "T2_PL_Swierk": "prod", - "T2_PL_Warsaw": "prod", - "T2_PT_NCG_Lisbon": "prod", - "T2_RU_IHEP": "prod", - "T2_RU_INR": "prod", - "T2_RU_ITEP": "prod", - "T2_RU_JINR": "prod", - "T2_TR_METU": "prod", - "T2_TW_NCHC": "prod", - "T2_UA_KIPT": "prod", - "T2_UK_London_Brunel": "prod", - "T2_UK_London_IC": "prod", - "T2_UK_SGrid_Bristol": "prod", - "T2_UK_SGrid_RALPP": "prod", - "T2_US_Caltech": "prod", - "T2_US_Florida": "prod", - "T2_US_MIT": "prod", - "T2_US_Nebraska": "prod", - "T2_US_Purdue": "prod", - "T2_US_UCSD": "prod", - "T2_US_Vanderbilt": "trans", - "T2_US_Wisconsin": "prod", - "T3_BG_UNI_SOFIA": "trans", - "T3_BY_NCPHEP": "nowhere", - "T3_CH_CERN_DOMA": "nowhere", - "T3_CH_CERN_OpenData": "nowhere", - "T3_CH_CMSAtHome": "nowhere", - "T3_CH_PSI": "trans", - "T3_CN_PKU": "nowhere", - "T3_FR_IPNL": "trans", - "T3_HR_IRB": "nowhere", - "T3_HU_Debrecen": "nowhere", - "T3_IN_TIFRCloud": "nowhere", - "T3_IN_VBU": "nowhere", - "T3_IR_IPM": "nowhere", - "T3_IT_Bologna": "trans", - "T3_IT_MIB": "trans", - "T3_IT_Perugia": "nowhere", - "T3_IT_Trieste": "trans", - "T3_KR_KISTI": "nowhere", - "T3_KR_KNU": "trans", - "T3_KR_UOS": "nowhere", - "T3_MX_Cinvestav": "nowhere", - "T3_RU_FIAN": "nowhere", - "T3_RU_MEPhI": "nowhere", - "T3_TH_CHULA": "nowhere", - "T3_TW_NCU": "nowhere", - "T3_TW_NTU_HEP": "trans", - "T3_UK_London_QMUL": "nowhere", - "T3_UK_London_RHUL": "nowhere", - "T3_UK_SGrid_Oxford": "nowhere", - "T3_UK_ScotGrid_GLA": "nowhere", - "T3_US_Baylor": "trans", - "T3_US_Brown": "trans", - "T3_US_Colorado": "trans", - "T3_US_FIT": "nowhere", - "T3_US_FNALLPC": "nowhere", - "T3_US_HEPCloud": "nowhere", - "T3_US_MIT": "nowhere", - "T3_US_Minnesota": "nowhere", - "T3_US_NERSC": "nowhere", - "T3_US_NotreDame": "nowhere", - "T3_US_OSG": "nowhere", - "T3_US_OSU": "nowhere", - "T3_US_Princeton_ICSE": "nowhere", - "T3_US_PuertoRico": "nowhere", - "T3_US_Rice": "trans", - "T3_US_Rutgers": "trans", - "T3_US_SDSC": "nowhere", - "T3_US_TAMU": "nowhere", - "T3_US_TTU": "nowhere", - "T3_US_UCR": "nowhere", - "T3_US_UMD": "trans", - "T3_US_UMiss": "nowhere", - "T3_US_UVA": "nowhere", - "T3_US_VC3_NotreDame": "nowhere", - "T3_US_Vanderbilt_EC2": "prod" -} \ No newline at end of file From c518de43f84e243738eac1b94570498748efee23 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 29 Jan 2020 13:41:37 -0600 Subject: [PATCH 052/192] Delete credentials.json --- AAAOps/AAA_Subscription/credentials.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100755 AAAOps/AAA_Subscription/credentials.json diff --git a/AAAOps/AAA_Subscription/credentials.json b/AAAOps/AAA_Subscription/credentials.json deleted file mode 100755 index 65d3dfa..0000000 --- a/AAAOps/AAA_Subscription/credentials.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "producer" : "cms_aaa_subscription", - "type" : "metric", - "type_prefix" : "raw", - "topic" : "cms.aaa.subscription", - "host_and_ports" : "cms-mb.cern.ch:61313", -} From 6a33969f0d20c7c62e95254f65a7feb1c55f196c Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 29 Jan 2020 14:22:20 -0600 Subject: [PATCH 053/192] Delete checkAdler32_forFilesAtSites copy.py --- scripts/checkAdler32_forFilesAtSites copy.py | 64 -------------------- 1 file changed, 64 deletions(-) delete mode 100644 scripts/checkAdler32_forFilesAtSites copy.py diff --git a/scripts/checkAdler32_forFilesAtSites copy.py b/scripts/checkAdler32_forFilesAtSites copy.py deleted file mode 100644 index e1ed21c..0000000 --- a/scripts/checkAdler32_forFilesAtSites copy.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -import os -import sys -import subprocess -import urllib -import json - -## This tool will get all replicas register for a particular LFN and will return the CMS sites + PFN + checksum (of the corrupt file). -## USAGE: python checkAdler32ForFilesAtSites.py - -def init(): - f = str(sys.argv[1]) - get_cksum_from_catalog(f) - get_file_replicas(f) - get_cksum_from_replica(f) - -def get_cksum_from_catalog(lfn): - url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/data?file=' + lfn - result = json.load(urllib.urlopen(url)) - for p in result['phedex']['dbs']: - for d in p['dataset']: - for b in d['block']: - for f in b['file']: - adler32.append(str(f['checksum']).split(',')[0].split(':')[1]) - -def get_file_replicas(lfn): - url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/filereplicas?lfn=' + lfn - result = json.load(urllib.urlopen(url)) - for b in result['phedex']['block']: - for f in b['file']: - for r in f['replica']: - sites.append(r['node']) - -def get_cksum_from_replica(lfn): - if len(adler32[0]) > 0: - print "-----------------------" - print "FILE: " + lfn - print "TMDB: " + adler32[0] - print "Replicas: (" + str(len(sites)) + ") " + str(sites).replace("u'","").replace("'","").replace("[","").replace("]","") - print "-----------------------" - if len(sites) == 0: - print "no corrupt files found" - else: - cf = 0 - for s in range(len(sites)): - url = 'https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn?protocol=srmv2&lfn=' + str(lfn) + '&node=' + str(sites[s]) - result = json.load(urllib.urlopen(url)) - for m in result['phedex']['mapping']: - pfn.append(m['pfn']) - cmd = "gfal-sum " + pfn[s] + " adler32" - result = subprocess.check_output(cmd, shell=True) - cksm = str(result).split('.root ')[1] - if str(adler32[0]).strip() != str(cksm).strip(): - print "site: " + sites[s] + " - " + result - cf = cf + 1 - print str(cf) + " site(s) with corrupt files found" - else: - print "no checksum info in TMDB for: " + lfn - -sites = [] -adler32 = [] -pfn = [] -corrupted = [] -init() From 119b54826a6ffaa7ea4d817ccd75efca1b0915f6 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Wed, 29 Jan 2020 14:49:21 -0600 Subject: [PATCH 054/192] Update setup.sh --- scripts/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup.sh b/scripts/setup.sh index adf0fcd..e3e3cc8 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -2,6 +2,6 @@ cwd=$PWD source ~/phedex/PHEDEX/etc/profile.d/env.sh cd ~/work/public/CMSSW_7_4_15/src cmsenv -source /cvmfs/cms.cern.ch/crab/crab.sh +source /cvmfs/cms.cern.ch/crab3/crab.sh voms-proxy-init -voms cms cd $cwd From 32abd6dc05544e16bf44b82ace79ce6b10aa4376 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Thu, 30 Jan 2020 09:37:19 -0600 Subject: [PATCH 055/192] Update dbs_phedex_consistency.py --- consistency_check/spark/dbs_phedex_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py index 9ba8a1a..269357a 100644 --- a/consistency_check/spark/dbs_phedex_consistency.py +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -41,7 +41,7 @@ def invalid_dbs_present_phedex(self): Returns a dataframe with datasets which have "INVALID" status in DBS and are "PRESENT" in phedex - :func: run_consistency.deleted_dbs_present_phedex() + :func: run_consistency.invalid_dbs_present_phedex() for reference dbs d_dataset_access_type_id: 1 : valid @@ -86,7 +86,7 @@ def valid_dbs_missing_phedex(self): ''' Returns a dataframe with block name which have "VALID" status in DBS and are "MISSING" in phedex - :func: run_consistency.deleted_dbs_present_phedex() + :func: run_consistency.valid_dbs_missing_phedex() ''' out = args.out_path + "/valid_dbs_missing_phedex" From 8d85077c5f25f048f172a61889eca891581cbc55 Mon Sep 17 00:00:00 2001 From: Pradeep <35749353+pradeepjasal@users.noreply.github.com> Date: Mon, 3 Feb 2020 14:19:55 -0600 Subject: [PATCH 056/192] Update README.md --- consistency_check/spark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md index ce638f1..7eb1392 100644 --- a/consistency_check/spark/README.md +++ b/consistency_check/spark/README.md @@ -21,4 +21,4 @@ login to lxplus7.cern.ch Example : --timestamp argument for checking the list for last n number of days -python filemismatch.py --timestamp 7 +python filemismatch.py --past_days 7 From b491faf225b5dec04814cac0d43b8fa5fc499fe6 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 14 Apr 2020 11:23:38 +0200 Subject: [PATCH 057/192] AAA scripts updated with new global redirectors host names --- AAAOps/Federation/create_allow-list.sh | 20 ++++++++++---------- AAAOps/Federation/exceptions.json | 7 +++++-- AAAOps/StorageBackend/checkVersion.py | 8 ++++---- AAAOps/XfedKibana/XRDFED-kibana-probe.py | 14 ++++++++++++-- AAAOps/XfedKibana/kibana_cron.sh | 3 +++ 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/AAAOps/Federation/create_allow-list.sh b/AAAOps/Federation/create_allow-list.sh index 63462b6..9e0ad1b 100755 --- a/AAAOps/Federation/create_allow-list.sh +++ b/AAAOps/Federation/create_allow-list.sh @@ -4,10 +4,10 @@ BASE=/root/ FEDINFO=/opt/TransferTeam/AAAOps/Federation/ export XRD_NETWORKSTACK=IPv4 -declare -a redirectors=("xrdcmsglobal01.cern.ch:1094" "xrdcmsglobal02.cern.ch:1094" "cms-xrd-transit.cern.ch:1094") +declare -a redirectors=("cms-xrd-global01.cern.ch:1094" "cms-xrd-global02.cern.ch:1094" "cms-xrd-transit.cern.ch:1094") for j in "${redirectors[@]}";do - if [ "$j" == "xrdcmsglobal01.cern.ch:1094" ] || [ "$j" == "xrdcmsglobal02.cern.ch:1094" ]; then + if [ "$j" == "cms-xrd-global01.cern.ch:1094" ] || [ "$j" == "cms-xrd-global02.cern.ch:1094" ]; then #query european reginal redirectors xrdmapc --list all "$j" | grep -E 'xrootd.ba.infn.it|xrootd-redic.pi.infn.it|llrxrd-redir.in2p3.fr:1094' | awk '{print $3}' | cut -d ':' -f1 > $BASE/tmp_euRED_$j xrdmapc --list all "$j" | grep -E 'cmsxrootd2.fnal.gov|xrootd.unl.edu' | awk '{print $3}' | cut -d ':' -f1 > $BASE/tmp_usRED_$j @@ -55,17 +55,17 @@ for j in "${redirectors[@]}";do done -diff $FEDINFO/in/prod_xrdcmsglobal01.cern.ch\:1094.txt $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt +diff $FEDINFO/in/prod_cms-xrd-global01.cern.ch\:1094.txt $FEDINFO/in/prod_cms-xrd-global02.cern.ch\:1094.txt stat=$(echo $?) if [ $stat == 1 ]; then - cat $FEDINFO/in/prod_xrdcmsglobal01.cern.ch\:1094.txt $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt | sort -u > $FEDINFO/in/prod.txt - cat $FEDINFO/out/list_eu_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_eu_xrdcmsglobal02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_eu.allow - cat $FEDINFO/out/list_us_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_us_xrdcmsglobal02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_us.allow + cat $FEDINFO/in/prod_cms-xrd-global01.cern.ch\:1094.txt $FEDINFO/in/prod_cms-xrd-global02.cern.ch\:1094.txt | sort -u > $FEDINFO/in/prod.txt + cat $FEDINFO/out/list_eu_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_eu_cms-xrd-global02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_eu.allow + cat $FEDINFO/out/list_us_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_us_cms-xrd-global02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_us.allow else - cp $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt $FEDINFO/in/prod.txt - cp $FEDINFO/out/list_us_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_us.allow - cp $FEDINFO/out/list_eu_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_eu.allow + cp $FEDINFO/in/prod_cms-xrd-global02.cern.ch\:1094.txt $FEDINFO/in/prod.txt + cp $FEDINFO/out/list_us_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_us.allow + cp $FEDINFO/out/list_eu_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_eu.allow fi @@ -87,6 +87,6 @@ rm $FEDINFO/in/tmp -cat $FEDINFO/in/prod.txt | sort -u | awk -F. '{if ($NF == "uk" && $(NF-2) != "rl" || $NF == "fr" || $(NF-1) == "cern" || $(NF-1) == "fnal" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "it" && $(NF-2) == "cnaf") print $(NF-4)"."$(NF-2); else if ( $NF == "it" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "uk" && $(NF-2) == "rl" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "kr") print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "be" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt +cat $FEDINFO/in/prod.txt | sort -u | awk -F. '{if ($NF == "uk" && $(NF-2) != "rl" || $NF == "fr" || $(NF-1) == "cern" || $(NF-1) == "fnal" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "it" && $(NF-2) == "cnaf") print $(NF-4)"."$(NF-2); else if ( $NF == "it" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-3) == "xrootd-vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "uk" && $(NF-2) == "rl" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "kr") print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "be" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt cat $FEDINFO/in/trans.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "kr" || $NF == "it" || $NF == "ch" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ($NF == "it" && $(NF-2) == "cnaf" ) print $(NF-4)"."$(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/trans_domain.txt diff --git a/AAAOps/Federation/exceptions.json b/AAAOps/Federation/exceptions.json index 833b28d..9cc0844 100644 --- a/AAAOps/Federation/exceptions.json +++ b/AAAOps/Federation/exceptions.json @@ -71,7 +71,7 @@ "VOname" : "T3_IT_Trieste" }, - "se11.accre.vanderbilt.edu" : { + "xrootd-vanderbilt.sites.opensciencegrid.org" : { "VOname" : "T2_US_Vanderbilt" }, @@ -99,10 +99,13 @@ "csc.fi" : { "VOname" : "T2_FI_HIP" }, - "eoscms-srv-m1.cern.ch" : { + "eoscms-qdb-250e2da131.cern.ch" : { "VOname" : "T2_CH_CERN" }, "cmsxrootd-2.gridka.de" : { "VOname" : "T1_DE_KIT" + }, + "cms-t2-se01.sdfarm.kr" : { + "VOname" : "T2_KR_KISTI" } } diff --git a/AAAOps/StorageBackend/checkVersion.py b/AAAOps/StorageBackend/checkVersion.py index 87f3c43..bb65ccc 100644 --- a/AAAOps/StorageBackend/checkVersion.py +++ b/AAAOps/StorageBackend/checkVersion.py @@ -4,7 +4,7 @@ with open("cms_site_names.txt", 'r') as f: - for site in f: - runCommand = "./storage_backend.sh "+site - p1 = Popen(runCommand, shell=True) - p1.wait() + for site in f: + runCommand = "./storage_backend.sh "+site + p1 = Popen(runCommand, shell=True) + p1.wait() diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py b/AAAOps/XfedKibana/XRDFED-kibana-probe.py index baa2ebe..fffd610 100755 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe.py +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe.py @@ -320,16 +320,26 @@ def main(): FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" services = { - "XRDFED_CMS-GLOBAL01":{'redirector':'xrdcmsglobal01.cern.ch:1094', + "XRDFED_CMS-GLOBAL01-NEW":{'redirector':'cms-xrd-global01.cern.ch:1094', 'file_below': FILEABOVE, 'file_above': FILEBELOW, 'extra_notes':CMSLINK}, - "XRDFED_CMS-GLOBAL02":{'redirector':'xrdcmsglobal02.cern.ch:1094', + "XRDFED_CMS-GLOBAL02-NEW":{'redirector':'cms-xrd-global02.cern.ch:1094', 'file_below': FILEABOVE, 'file_above': FILEBELOW, 'extra_notes':CMSLINK}, +# "XRDFED_CMS-GLOBAL01":{'redirector':'xrdcmsglobal01.cern.ch:1094', +# 'file_below': FILEABOVE, +# 'file_above': FILEBELOW, +# 'extra_notes':CMSLINK}, + +# "XRDFED_CMS-GLOBAL02":{'redirector':'xrdcmsglobal02.cern.ch:1094', +# 'file_below': FILEABOVE, +# 'file_above': FILEBELOW, +# 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', 'file_below': FILEABOVE, 'file_above': FILEBELOW, diff --git a/AAAOps/XfedKibana/kibana_cron.sh b/AAAOps/XfedKibana/kibana_cron.sh index d11bdef..22e2627 100755 --- a/AAAOps/XfedKibana/kibana_cron.sh +++ b/AAAOps/XfedKibana/kibana_cron.sh @@ -6,6 +6,9 @@ /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-US.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-EU.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log +/usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL01-NEW.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log +/usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL02-NEW.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log + /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL01.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL02.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-EU-LLR.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log From b086477b678601facf741c9335bdc46d584a8ff5 Mon Sep 17 00:00:00 2001 From: dmielaikaite Date: Tue, 14 Apr 2020 12:16:04 +0200 Subject: [PATCH 058/192] send aaa data to monit --- AAAOps/send_aaa_matrics/aaa_federation.py | 115 ++++++++++++++++++++++ AAAOps/send_aaa_matrics/aaa_federation.sh | 9 ++ 2 files changed, 124 insertions(+) create mode 100755 AAAOps/send_aaa_matrics/aaa_federation.py create mode 100755 AAAOps/send_aaa_matrics/aaa_federation.sh diff --git a/AAAOps/send_aaa_matrics/aaa_federation.py b/AAAOps/send_aaa_matrics/aaa_federation.py new file mode 100755 index 0000000..13c781c --- /dev/null +++ b/AAAOps/send_aaa_matrics/aaa_federation.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +import os +import sys +import subprocess +import urllib +import json +from datetime import datetime +import collections +import stomp +import argparse +from CMSMonitoring.StompAMQ import StompAMQ +import uuid +from itertools import islice +import hashlib +import stomp +from uuid import uuid4 +from urllib.request import urlopen +import time + +print("stomp version", stomp.__version__) + + +class OptionParser(): + def __init__(self): + "User based option parser" + self.parser = argparse.ArgumentParser() + msg = "Send aaa subscription json file via StompAMQ to a broker, provide broker credentials in JSON file" + self.parser.add_argument("--amq", action="store", + dest="amq", default="", help=msg, required=True) + + +class MyListener(stomp.ConnectionListener): + def on_error(self, headers, message): + print('received an error "%s"' % message) + + def on_message(self, headers, message): + print('received a message "%s"' % message) + + +def credentials(fname=None): + "Read credentials from MONIT_BROKER environment" + if not fname: + fname = os.environ.get('MONIT_BROKER', '') + if not os.path.isfile(fname): + raise Exception( + "Unable to locate MONIT credentials, please setup MONIT_BROKER") + return {} + with open(fname, 'r') as istream: + data = json.load(istream) + return data + + +def parseFederationJson(): + ''' + modifies federation.json with tags and + Return json object to feed cern metris + ''' + url = 'http://vocms039.cern.ch/aaa-fedinfo/federations.json' + result = json.load(urlopen(url)) + fed_data = createFlattenJson(result) + f = open("fed.json", "r") + for x in f: + data = json.loads(x) + yield data + +def createFlattenJson(json_file): + ''' + flattens the federation.json file + ''' + for k,v in json_file.items(): + if ("prod" in k): + statusCode = 1 + stateName = "Production Federation" + elif ("trans" in k): + statusCode = 2 + stateName = "Transitional Federation" + else: + statusCode = 3 + stateName = "not at any federation" + for site in v: + my_json = ('{"siteName" : "%s", "state" : "%s", "statusCode" : %s, "stateName" : "%s"}' %(site, k, statusCode, stateName)) + with open("fed.json","a") as f: + f.write("%s\n" %my_json) + +if __name__ == "__main__": + optmgr = OptionParser() + opts = optmgr.parser.parse_args() + payload = parseFederationJson() + if opts.amq: + creds = credentials(opts.amq) + username = creds['username'] + password = creds['password'] + topic = creds['topic'] + host, port = creds['host_and_ports'].split(':') + port = int(port) + producer = creds['producer'] + topic = creds['topic'] + hosts = [(host, port)] + if creds and StompAMQ: + amq = StompAMQ(username, password, producer, topic, key=None, + cert=None, validation_schema=None, host_and_ports=[(host, port)]) + eod = False + wait_seconds = 10 + while not eod: + messages = [] + for d in payload: + notif,_,_ = amq.make_notification(d, "aaa_federations_document", dataSubfield=None) + messages.append(notif) + if messages: + print(messages) + amq.send(messages) + time.sleep(wait_seconds) + else: + eod = True diff --git a/AAAOps/send_aaa_matrics/aaa_federation.sh b/AAAOps/send_aaa_matrics/aaa_federation.sh new file mode 100755 index 0000000..8ab5812 --- /dev/null +++ b/AAAOps/send_aaa_matrics/aaa_federation.sh @@ -0,0 +1,9 @@ +#!/bin/bash + + +set -x +export PYTHONPATH=$PYTHONPATH:/opt/TransferTeam/AAAOps/send_aaa_matrics/CMSMonitoring/src/python/ +export PYTHONPATH=$PYTHONPATH:/opt/TransferTeam/AAAOps/send_aaa_matrics/stomp.py/stomp/ +#source /cvmfs/sft.cern.ch/lcg/views/LCG_96python3/x86_64-centos7-gcc8-opt/setup.sh +rm fed.json +python3 /opt/TransferTeam/AAAOps/send_aaa_matrics/aaa_federation.py --amq /opt/TransferTeam/AAAOps/send_aaa_matrics/credentials.json From 9b107c6064b01a080ecf0fee6e08e3ec4804946f Mon Sep 17 00:00:00 2001 From: Donata Mielaikaite Date: Wed, 13 May 2020 15:28:18 +0200 Subject: [PATCH 059/192] modified filemismatch --- consistency_check/spark/filemismatch.py | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 consistency_check/spark/filemismatch.py diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py new file mode 100644 index 0000000..5647105 --- /dev/null +++ b/consistency_check/spark/filemismatch.py @@ -0,0 +1,71 @@ +#!/usr/bin/env spark-submit +from __future__ import print_function +import argparse +import json +import os + +from pyspark import SparkConf, SparkContext, StorageLevel +from pyspark.sql import SparkSession, Column +from pyspark.sql.functions import col +import pyspark.sql.functions as fn +import pyspark.sql.types as types +from pyspark.sql import DataFrame +import datetime +import time + + +# stolen from CMSSpark +import schemas + +class OptionParser(): + + """Dataset access types: + 81,DELETED + 42,DEPRECATED + 2,INVALID + 41,PRODUCTION + 1,VALID + """ + + def __init__(self): + "option Parser" + self.parser = argparse.ArgumentParser(prog='consistency') + msg = "Files mismatch since last n days : default in past 7 days" + self.parser.add_argument("--past_days",action="store",dest="days",default=7,help=msg,type=int) + msg= "Output path in HDFS for result" + self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) + +def fileMismatch(args): + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + spark = SparkSession(sc) + print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) + + avroreader = spark.read.format("com.databricks.spark.avro") + csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") + dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") + dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + + current = time.time() + past_n_days = args.days + delta_t = current - past_n_days*60*60*24 + delta_t_str = str(delta_t) + delta_t = delta_t_str[:10] + + if args.out_path: + mismatch_df = (dbs_files + .filter(col('f_is_file_valid') == '0') + .filter(col('f_last_modification_date') > delta_t) + .join(dbs_datasets,col('f_dataset_id') == col('d_dataset_id')) + .filter((col('d_dataset_access_type_id') == '1') | (col('d_dataset_access_type_id') == '41')) + .filter(col('f_logical_file_name').isNotNull()) + .where(~(dbs_files.f_last_modified_by.contains('dmielaik') | dbs_files.f_last_modified_by.contains('ogarzonm'))) + .select('d_dataset','f_last_modified_by','f_logical_file_name') + .distinct()) + mismatch_df.select('f_logical_file_name','f_last_modified_by').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(args.out_path) + mismatch_df.groupby('d_dataset').agg(fn.count(fn.col("f_logical_file_name")).alias('extra_lfn_phedex')).show() +if __name__ == '__main__': + optmgr = OptionParser() + args = optmgr.parser.parse_args() + fileMismatch(args) + From 4d0d9d1186d13082d015661a18e46f5bb43a7da6 Mon Sep 17 00:00:00 2001 From: Donata Mielaikaite Date: Wed, 13 May 2020 15:46:34 +0200 Subject: [PATCH 060/192] fixed filemismatch.py --- consistency_check/spark/filemismatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py index 180c4fa..5ffaa49 100644 --- a/consistency_check/spark/filemismatch.py +++ b/consistency_check/spark/filemismatch.py @@ -18,7 +18,7 @@ import schemas class OptionParser(): - """Dataset access types: + """Dataset access types: 81,DELETED 42,DEPRECATED 2,INVALID From 4aa90fff4fecf9072767c4ed4779eb1ed1debf31 Mon Sep 17 00:00:00 2001 From: FernandoGarzon Date: Wed, 15 Jul 2020 15:55:57 +0200 Subject: [PATCH 061/192] Old kibana probe fixed --- AAAOps/XfedKibana/XRDFED-kibana-probe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py b/AAAOps/XfedKibana/XRDFED-kibana-probe.py index fffd610..e924123 100755 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe.py +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe.py @@ -151,6 +151,8 @@ def run_xrd_commands(cmd,args): "-DIConnectTimeout","30", "-DITransactionTimeout","60", "-DIRequestTimeout","60" ] + args + if not out: + out = "1" try: start = time.time() proc = subprocess.Popen(xrd_args, From c2e921cf63d60840a8e30604b4eaa500441e3476 Mon Sep 17 00:00:00 2001 From: Fernando Garzon Date: Tue, 20 Oct 2020 09:52:09 -0500 Subject: [PATCH 062/192] Kibana probe with latest changes applied --- .../XRDFED-kibana-probe_JSON.py | 310 ++++++++++++++++++ AAAOps/XfedKibana_JSON/kibana_probe.sh | 2 + AAAOps/XfedKibana_JSON/single_quotes.sh | 2 + 3 files changed, 314 insertions(+) create mode 100644 AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py create mode 100644 AAAOps/XfedKibana_JSON/kibana_probe.sh create mode 100644 AAAOps/XfedKibana_JSON/single_quotes.sh diff --git a/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py new file mode 100644 index 0000000..2324222 --- /dev/null +++ b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py @@ -0,0 +1,310 @@ +#!/usr/bin/python +# functional probe and SLS extractor for the "federation" xroot services +# highlights: +# - stateless (i.e. run from cron whenever needed) +# - will try to prevent parallel runs via lockfile +# - multithreaded, one thread per service to be tested +# - overall runtime cap at 10min +# - could extract some statistics from xroot directly, but these are ever-increasing counters +# Problems: +# - need to update the code whenever a service is addded/deleted/changed +# - uses "random" files on various Xroot services all over the world, these are (for now) the same as used by the experiments but these might change.. +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import json +import shutil +html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here +LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' +class Alarm(Exception): + pass +def alarm_handler(signum, frame): + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + raise Alarm +def clear_lock(): + try: + os.unlink(LOCKFILE) + except Exception,e: + print "could not remove lockfile:"+str(e) +def env_setup(): + os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' + os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' + os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + os.environ['KRB5CCNAME']='FILE:/dev/null' + os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" +def get_proxy(): + dev_null = open('/dev/null', 'rw') + (proxyfd,proxy)=tempfile.mkstemp(prefix='x509_xrdfed_',suffix='.pem') + os.close(proxyfd) + os.environ['X509_USER_PROXY']=proxy + ret = subprocess.call(['grid-proxy-init','-pwstdin'],stdin=dev_null,) + if ret > 0: + raise Exception("Cannot get X509 proxy") + dev_null.close() +def cleanup_proxy(): + try: + os.unlink(os.environ['X509_USER_PROXY']) + except Exception,e: + print "could not remove proxy file:"+str(e) +def try_lock(): + ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) + if ret > 0: + print "could not create lockfile" + return False + return True +def prepare_dictionary(servicename,redirector): + (errtext,version,out) = xrd_info(redirector) + dic={'service':servicename, 'host': redirector[:redirector.find(':')]} + if(errtext): + dic['version'] = 'unavailable' + dic['status'] = 'unavailable' + errtext = errtext.replace("'", "") + errtext = errtext.replace('"', '') + dic['comment'] = "Error getting info from redirector: "+errtext + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + else: + dic['version'] = version + return dic +def xrdcp_test(redirector,file): + (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", + ["-d","1", + "-f", + "-DIReadCacheSize","0", + "-DIRedirCntTimeout","180", + "root://"+redirector+'/'+file, + '/dev/null']) + return (errtext,err,elapsed) +def xrd_info(redirector): + version = "(unknown)" + (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", + [redirector, + "query","config", # 1:kXR_QStats + "version"]) # a_ll stats + if not out: + errtext = '' + os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") + os.system("head -n 1 /root/aux.txt > /root/aux2.txt") + f = open('/root/aux2.txt', 'r') + version = f.read() + if not version: + version = "(unknown)" + else: + version = version[:-1] + else: + if not errtext: + try: + dom = xml.dom.minidom.parseString(out) + root_node = dom.documentElement + if root_node.tagName == 'statistics': + v_attr = root_node.getAttributeNode('ver') + version = v_attr.nodeValue + except Exception,e: + errtext = "ERROR: cannot parse answer:"+str(e) + return (errtext,version,out) +def run_xrd_commands(cmd,args): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + "-DIConnectTimeout","30", + "-DITransactionTimeout","60", + "-DIRequestTimeout","60" ] + args + err = '' + out = '' + try: + ran_try = True + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + ret = proc.returncode + elapsed = (time.time() - start) + err_redir_index = err.rfind('Received redirection to') + err_index3010 = err.rfind('(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind('(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + errtxt = '' + else: + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind('Last server error') + if err_index >= 0: + err_end_index=err.find("\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except Exception,e: + errtext = errtxt + "Exception: "+str(e) + dev_null.close() + return (errtxt,out,err,elapsed) +def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): + servicename=servicename.upper() + notes_text = "Redirector: "+redirector + availability = 'Available' + availinfo = '' + c = '' + if 'status' in dicci and dicci['status'] == 'unavailable': + pass + elif file_below == None and file_above == None: + availability = 'Unavialable' + c = 'Non-existing File Above and File Below.' + dicci['xrdcp_below_time'] = 0 + dicci['xrdcp_below_time'] = 0 + else: + if (file_below): + notes_text = notes_text + "File below: " + file_below + (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) + if err_below: + availability = 'Degraded' + #availinfo=availinfo+" Error below redirector "+err_below + dump_sane = re.sub('---*','__',dump_below) + c = c+"Error for file BELOW: "+err_below+". Dumpsane: "+dump_sane+ '.' + dicci['xrdcp_below_time'] = 0 + else: + #availinfo=availinfo+" File below: OK " + dicci['xrdcp_below_time'] = elapsed_below + else: + c = "Error for file BELOW: Non-existing File Below. " + dicci['xrdcp_below_time'] = 0 + if(file_above): + notes_text = notes_text + "File elsewhere: " + file_above + (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) + if err_above : + availability = 'Degraded' + #availinfo=availinfo+" Error above redirector "+err_above + dump_sane = re.sub('---*','__',dump_above) + c = c+"Error for file ABOVE: "+err_above+". Dumpsane: "+dump_sane+'.' + dicci['xrdcp_above_time'] = 0 + else: + #availinfo = availinfo+" File above: OK " + dicci['xrdcp_above_time'] = elapsed_above + else: + c = c + "Error for file ABOVE: Non-existing File Above." + dicci['xrdcp_above_time'] = 0 + #availinfo = availinfo + " " + notes_text + dicci['status']= str(availability) + if c == '': + c = 'N/A' + c = c.replace("\n", "") + c = c.replace("\r", "") + dicci ['Comment'] = c + with open(html_dir +'KIBANA_PROBES.json', 'a') as f: + json.dump(dicci, f) + f.write('\n') +def main(): + debug = 0 + atexit.register(clear_lock) + if len(sys.argv) > 1: + if sys.argv[1] == '-d': + debug=1 + if not try_lock(): + sys.exit(1) + if not os.path.exists(html_dir): + os.makedirs(html_dir) + env_setup() + timeout_sec = 10 * 60 # limit overall runtime to 10min + signal.signal(signal.SIGALRM, alarm_handler) + ATLASLINK="%BR%Monitoring:%BR%\n http://atl-prod07.slac.stanford.edu:8080/display?page=xrd_report/aggregated/total_xrootd_lgn %BR%\n http://dashb-atlas-xrootd-transfers.cern.ch/ui %BR%\nhttp://dashb-atlas-ssb.cern.ch/dashboard/request.py/siteview#currentView=FAX+redirectors&highlight=false %BR%\n" + CMSLINK="%BR%Monitoring:%BR%\n http://xrootd.t2.ucsd.edu/dashboard/ %BR%\n http://dashb-cms-xrootd-transfers.cern.ch/ui %BR%\n" + FILEABOVE="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + services = { + "XRDFED_CMS-GLOBAL01-NEW":{'redirector':'cms-xrd-global01.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL02-NEW":{'redirector':'cms-xrd-global02.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-UNL":{'redirector':'xrootd.unl.edu:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-PISA":{'redirector':'xrootd-redic.pi.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL":{'redirector':'cms-xrd-global.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US":{'redirector':'cmsxrootd.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU":{'redirector':'xrootd-cms.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-IPv6":{ 'redirector':'xrootd-cms-redir-01.cr.cnaf.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT":{'redirector':'cms-xrd-transit.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT01":{'redirector':'vocms031.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT02":{'redirector':'vocms032.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + } + signal.alarm(timeout_sec) + try: + diccionaries = [] + for xrd in services: + services[xrd].update(servicename=xrd) + servicename = xrd + dicci = prepare_dictionary(servicename, services[xrd]['redirector']) + diccionaries.append(dicci) + for dicci in diccionaries: + service = dicci['service'] + argus = services[service] + argus['dicci'] = dicci + if debug: + test_redirector(** services[xrd]) + else: + t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" + t.start() + #t.join() + #os.system('source ~/single_quotes.sh') + except Alarm: + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + signal.alarm(0) +if __name__ == '__main__': + for file in os.listdir(html_dir): + if file == 'KIBANA_PROBES.json': + os.remove(html_dir+'KIBANA_PROBES.json') + break + main() diff --git a/AAAOps/XfedKibana_JSON/kibana_probe.sh b/AAAOps/XfedKibana_JSON/kibana_probe.sh new file mode 100644 index 0000000..39c1407 --- /dev/null +++ b/AAAOps/XfedKibana_JSON/kibana_probe.sh @@ -0,0 +1,2 @@ +python ./XRDFED-kibana-probe_JSON.py +source ./single_quotes.sh diff --git a/AAAOps/XfedKibana_JSON/single_quotes.sh b/AAAOps/XfedKibana_JSON/single_quotes.sh new file mode 100644 index 0000000..8afd322 --- /dev/null +++ b/AAAOps/XfedKibana_JSON/single_quotes.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sed -i "y/\"/'/" /var/www/html/aaa-probe/KIBANA_PROBES.json From fff3f5259cf90ea4fe07a946a0982ee7ff8e9656 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 9 Aug 2021 18:31:40 +0200 Subject: [PATCH 063/192] Add the Redirector Service Availability probe and uploader --- .../README.initial.work | 113 ++++++ .../RedirectorServiceAvailability/README.md | 19 + .../XRDFED-kibana-probe_JSON_General.py | 352 ++++++++++++++++++ .../logs/XRDFED_probe_json.log | 15 + .../logs/XRDFED_send.log | 2 + .../send_metrics.py | 83 +++++ .../uploadmetricGeneral.sh | 36 ++ 7 files changed, 620 insertions(+) create mode 100644 AAAOps/RedirectorServiceAvailability/README.initial.work create mode 100644 AAAOps/RedirectorServiceAvailability/README.md create mode 100644 AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py create mode 100644 AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log create mode 100644 AAAOps/RedirectorServiceAvailability/logs/XRDFED_send.log create mode 100644 AAAOps/RedirectorServiceAvailability/send_metrics.py create mode 100755 AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh diff --git a/AAAOps/RedirectorServiceAvailability/README.initial.work b/AAAOps/RedirectorServiceAvailability/README.initial.work new file mode 100644 index 0000000..11e0c70 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/README.initial.work @@ -0,0 +1,113 @@ +Failsafe Redirector Probe: Task 5 in AAA Tasks and Milestones~Ref[1]  + + +The probe is running on vocms036 as a root cronscript. + + +[1] Fernando’s Original Version of the Probe for all redirectors + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py + + +[2] Fernando’s Failsafe Redirector Probe for one redirector + +This one is implemented by Fernando with a PING check: + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON_1_TEST.py + + +[3] Fernando’s cron script + +The cron is running every 15min. The cron script, /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetric.sh, + +executes the probe for each redirector unlike the one in [1] and uploads the metric to the data source monit_idb_cmsaaa which the Grafana queries to create the Redirector Probe Dashboard~Ref[2] (Service Availability Dashboard) as follows: + +python XRDFED-kibana-probe_JSON_1.py + +python3 send_metrics.py > $logs/XRDFED_send.log 2>&1 + +The first line probes one redirector and creates one json file + +The second line sends the json file to the MONIT data source monit_idb_cmsaaa. + +These two lines are repeated for all redirectors (14 of them as of Aug. 5, 2021). + + +[4] Failsafe Redirector Probe + +The probe in [1] was taken and the ping check in [2] is added: + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON_General.py + + +For the redirectors that fail a ping command, the probe produces an output similar to this json: + +{"status": "unavailable", "comment": "Unpingable host", "service": "XRDFED_CMS-GLOBAL01-NEW", "host": "cmxrd-global01.cern.ch", "version": "unavailable", "xrdcp_above_time": 0, "xrdcp_below_time": 0, "Comment": "N/A"} + +The json is sent as the probed metrics to the MONIT data source. + +For the redirectors that can be ping-able, the probe checks the needed metrics to send to the MONIT data source. + + +In the end, the python probe produces a file that contains 14 lines of metrics in the json format. + +The execution time of the probe is typically around 2min. + + +[5] The metric uploader + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/send_metrics.py sends the output produced by the python probe. The send_metrics.py python script is the same and used without any change. + + +[6] Cron Script + +The cron script is renamed to: + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetricGeneral.sh + +The script basically contains these two lines of commands: + +python /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 + +python3 /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/send_metrics.py > $logs/XRDFED_send.log 2>&1 + + +[7] Puppetized Cron + +The cron can be puppetized by adding a cron entry that contains various crontab parameters for the necessary cron script,  /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetricGeneral.sh + +, execution using https://gitlab.cern.ch/ai/it-puppet-hostgroup-vocms/-/blob/qa/code/manifests/xrootd/monitor.pp  + +After the puppet agent is run, an entry in the crontab as follows: + +\# Puppet Name: AAA Service Availability Grafana Dashboard metric upload + +*/15 * * * * /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetricGeneral.sh > /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/logs/uploadmetricGeneral.log 2>&1 + +will appear. + +[8] Gitlab +https://gitlab.cern.ch/bockjoo/aaa-monitoring/-/tree/master/RedirectorServiceAvailability + + +[9] Questions + +Do we need to send an alert email for extendedly unavailable redirectors? + +How to define ‘extendedly unavailable redirectors’? + +Is the Dashboard good enough for the service availability? + + +[10] Summary + +The work started by Fernando is completed by merging the ping check capability and consolidating the probe python code in a single file. The Grafana shows the Dashboard correctly. + + + +References +[0] ... +[1] ... +[2] https://monit-grafana.cern.ch/d/serviceAvailability/overview-service-availability?orgId=11&var-category=All + +[3] https://gitlab.cern.ch/ai/it-puppet-hostgroup-vocms/-/blob/qa/code/manifests/xrootd/monitor.pp diff --git a/AAAOps/RedirectorServiceAvailability/README.md b/AAAOps/RedirectorServiceAvailability/README.md new file mode 100644 index 0000000..93a4795 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/README.md @@ -0,0 +1,19 @@ +# Service Availabilty Probe and Uploader for Xrootd Redirectors + +Introduction +--- + +- This directory contains the xrootd redirector probe and the metric uploader. +- Currently the probe and the uploader run on one of the AAA vocms machines +- Details can be viewed from https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsCentralServicesXrootd and https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsAAAOperationsGuide#Monitoring + +Deploying the probe, the uploader, and the cron +--- +In the VM machine, as the root user, the github repository can be checked out under /opt/TransferTeam. +The crontab is managed by the puppet: https://gitlab.cern.ch/ai/it-puppet-hostgroup-vocms/-/blob/qa/code/manifests/xrootd/monitor.pp + +Operations +--- + +The probe and the uploader are executed every 15 minute by cron. +The uploaded metric is visualized in the Grafana dashboard: https://monit-grafana.cern.ch/goto/Q2CH71M7z diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py new file mode 100644 index 0000000..a1a61f0 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -0,0 +1,352 @@ +#!/usr/bin/python +# functional probe and SLS extractor for the "federation" xroot services +# highlights: +# - stateless (i.e. run from cron whenever needed) +# - will try to prevent parallel runs via lockfile +# - multithreaded, one thread per service to be tested +# - overall runtime cap at 10min +# - could extract some statistics from xroot directly, but these are ever-increasing counters +# Problems: +# - need to update the code whenever a service is addded/deleted/changed +# - uses "random" files on various Xroot services all over the world, these are (for now) the same as used by the experiments but these might change.. +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +#import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import json +import shutil +import multiprocessing +html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here +#Bockjoo Uncomment and comment a line below +#LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' +LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe-general' +# Bockjoo Uncomment and comment a line below +#probes_json='KIBANA_PROBES.json' +probes_json='KIBANA_PROBES_GENERAL.json' +timeout_sec = 10 * 60 +class Alarm(Exception): + pass +def alarm_handler(signum, frame): + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + raise Alarm +def clear_lock(): + try: + os.unlink(LOCKFILE) + except Exception,e: + print "could not remove lockfile:"+str(e) +def env_setup(): + os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' + os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' + os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + os.environ['KRB5CCNAME']='FILE:/dev/null' + os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" +def get_proxy(): + dev_null = open('/dev/null', 'rw') + (proxyfd,proxy)=tempfile.mkstemp(prefix='x509_xrdfed_',suffix='.pem') + os.close(proxyfd) + os.environ['X509_USER_PROXY']=proxy + ret = subprocess.call(['grid-proxy-init','-pwstdin'],stdin=dev_null,) + if ret > 0: + raise Exception("Cannot get X509 proxy") + dev_null.close() +def cleanup_proxy(): + try: + os.unlink(os.environ['X509_USER_PROXY']) + except Exception,e: + print "could not remove proxy file:"+str(e) +def try_lock(): + ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) + if ret > 0: + print "could not create lockfile" + return False + return True + +def check_ping(hostname): + with open(os.devnull, 'w') as DEVNULL: + try: + subprocess.check_call( + ['ping', '-c', '3', hostname], + stdout=DEVNULL, # suppress output + stderr=DEVNULL + ) + is_up = True + except subprocess.CalledProcessError: + is_up = False + return is_up + +def prepare_dictionary(servicename,redirector): + is_up = check_ping(redirector[:redirector.find(':')]) + dic={'service':servicename, 'host': redirector[:redirector.find(':')]} + print(servicename," is up ",is_up) + if is_up == False: + dic['version'] = 'unavailable' + dic['status'] = 'unavailable' + dic['comment'] = "Unpingable host" + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + else: + (errtext,version,out) = xrd_info(redirector) + if(errtext): + dic['version'] = 'unavailable' + dic['status'] = 'unavailable' + errtext = errtext.replace("'", "") + errtext = errtext.replace('"', '') + dic['comment'] = "Error getting info from redirector: "+errtext + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + else: + dic['version'] = version + return dic +def xrdcp_test(redirector,file): + (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", + ["-d","1", + "-f", + "-DIReadCacheSize","0", + "-DIRedirCntTimeout","180", + "root://"+redirector+'/'+file, + '/dev/null']) + return (errtext,err,elapsed) +def xrd_info(redirector): + version = "(unknown)" + (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", + [redirector, + "query","config", # 1:kXR_QStats + "version"]) # a_ll stats + if not out: + errtext = '' + os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") + os.system("head -n 1 /root/aux.txt > /root/aux2.txt") + f = open('/root/aux2.txt', 'r') + version = f.read() + if not version: + version = "(unknown)" + else: + version = version[:-1] + else: + if not errtext: + try: + dom = xml.dom.minidom.parseString(out) + root_node = dom.documentElement + if root_node.tagName == 'statistics': + v_attr = root_node.getAttributeNode('ver') + version = v_attr.nodeValue + except Exception,e: + errtext = "ERROR: cannot parse answer:"+str(e) + return (errtext,version,out) +def run_xrd_commands(cmd,args): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + "-DIConnectTimeout","30", + "-DITransactionTimeout","60", + "-DIRequestTimeout","60" ] + args + err = '' + out = '' + try: + ran_try = True + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + ret = proc.returncode + elapsed = (time.time() - start) + err_redir_index = err.rfind('Received redirection to') + err_index3010 = err.rfind('(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind('(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + errtxt = '' + else: + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind('Last server error') + if err_index >= 0: + err_end_index=err.find("\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except Exception,e: + errtext = errtxt + "Exception: "+str(e) + dev_null.close() + return (errtxt,out,err,elapsed) +def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): + servicename=servicename.upper() + notes_text = "Redirector: "+redirector + availability = 'Available' + availinfo = '' + c = '' + if 'status' in dicci and dicci['status'] == 'unavailable': + availability = dicci['status'] + #pass + elif file_below == None and file_above == None: + availability = 'Unavialable' + c = 'Non-existing File Above and File Below.' + dicci['xrdcp_below_time'] = 0 + dicci['xrdcp_below_time'] = 0 + else: + if (file_below): + notes_text = notes_text + "File below: " + file_below + (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) + if err_below: + availability = 'Degraded' + #availinfo=availinfo+" Error below redirector "+err_below + dump_sane = re.sub('---*','__',dump_below) + c = c+"Error for file BELOW: "+err_below+". Dumpsane: "+dump_sane+ '.' + dicci['xrdcp_below_time'] = 0 + else: + #availinfo=availinfo+" File below: OK " + dicci['xrdcp_below_time'] = elapsed_below + else: + c = "Error for file BELOW: Non-existing File Below. " + dicci['xrdcp_below_time'] = 0 + if(file_above): + notes_text = notes_text + "File elsewhere: " + file_above + (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) + if err_above : + availability = 'Degraded' + #availinfo=availinfo+" Error above redirector "+err_above + dump_sane = re.sub('---*','__',dump_above) + c = c+"Error for file ABOVE: "+err_above+". Dumpsane: "+dump_sane+'.' + dicci['xrdcp_above_time'] = 0 + else: + #availinfo = availinfo+" File above: OK " + dicci['xrdcp_above_time'] = elapsed_above + else: + c = c + "Error for file ABOVE: Non-existing File Above." + dicci['xrdcp_above_time'] = 0 + #availinfo = availinfo + " " + notes_text + dicci['status']= str(availability) + if c == '': + c = 'N/A' + c = c.replace("\n", "") + c = c.replace("\r", "") + dicci ['Comment'] = c + with open(html_dir + probes_json, 'a') as f: + json.dump(dicci, f) + f.write('\n') +def main(): + debug = 0 + atexit.register(clear_lock) + if len(sys.argv) > 1: + if sys.argv[1] == '-d': + debug=1 + if not try_lock(): + sys.exit(1) + if not os.path.exists(html_dir): + os.makedirs(html_dir) + env_setup() + timeout_sec = 10 * 60 # limit overall runtime to 10min + signal.signal(signal.SIGALRM, alarm_handler) + ATLASLINK="%BR%Monitoring:%BR%\n http://atl-prod07.slac.stanford.edu:8080/display?page=xrd_report/aggregated/total_xrootd_lgn %BR%\n http://dashb-atlas-xrootd-transfers.cern.ch/ui %BR%\nhttp://dashb-atlas-ssb.cern.ch/dashboard/request.py/siteview#currentView=FAX+redirectors&highlight=false %BR%\n" + CMSLINK="%BR%Monitoring:%BR%\n http://xrootd.t2.ucsd.edu/dashboard/ %BR%\n http://dashb-cms-xrootd-transfers.cern.ch/ui %BR%\n" + FILEABOVE="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + services = { + "XRDFED_CMS-GLOBAL01":{'redirector':'cms-xrd-global01.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL02":{'redirector':'cms-xrd-global02.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-UNL":{'redirector':'xrootd.unl.edu:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-PISA":{'redirector':'xrootd-redic.pi.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL":{'redirector':'cms-xrd-global.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US":{'redirector':'cmsxrootd.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU":{'redirector':'xrootd-cms.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-IPv6":{ 'redirector':'xrootd-cms-redir-int.cr.cnaf.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT":{'redirector':'cms-xrd-transit.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT01":{'redirector':'vocms031.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT02":{'redirector':'vocms032.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK} + } + signal.alarm(timeout_sec) + try: + diccionaries = [] + for xrd in services: + services[xrd].update(servicename=xrd) + servicename = xrd + dicci = prepare_dictionary(servicename, services[xrd]['redirector']) + diccionaries.append(dicci) + for dicci in diccionaries: + service = dicci['service'] + argus = services[service] + argus['dicci'] = dicci + if debug: + test_redirector(** services[xrd]) + else: + t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" + t.start() + #t.join() + #os.system('source ~/single_quotes.sh') + #print(diccionaries) + except Alarm: + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + print "Check ",html_dir+probes_json + signal.alarm(0) +if __name__ == '__main__': + for file in os.listdir(html_dir): + if file == probes_json: + os.remove(html_dir+probes_json) + break + #p = multiprocessing.Process(target=main) + #p.start() + #p.join(180) + + #if p.is_alive(): + # print"killing process" + # os._exit(1) + #p.terminate() + #p.join() + main() diff --git a/AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log b/AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log new file mode 100644 index 0000000..1571b27 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log @@ -0,0 +1,15 @@ +('XRDFED_CMS-EU-BARI', ' is up ', True) +('XRDFED_CMS-TRANSIT01', ' is up ', True) +('XRDFED_CMS-US-FNAL', ' is up ', True) +('XRDFED_CMS-GLOBAL01', ' is up ', True) +('XRDFED_CMS-GLOBAL02', ' is up ', True) +('XRDFED_CMS-TRANSIT', ' is up ', True) +('XRDFED_CMS-US-UNL', ' is up ', True) +('XRDFED_CMS-EU-PISA', ' is up ', True) +('XRDFED_CMS-US', ' is up ', True) +('XRDFED_CMS-EU-LLR', ' is up ', True) +('XRDFED_CMS-EU-IPv6', ' is up ', True) +('XRDFED_CMS-EU', ' is up ', True) +('XRDFED_CMS-GLOBAL', ' is up ', True) +('XRDFED_CMS-TRANSIT02', ' is up ', True) +Check /var/www/html/aaa-probe/KIBANA_PROBES_GENERAL.json diff --git a/AAAOps/RedirectorServiceAvailability/logs/XRDFED_send.log b/AAAOps/RedirectorServiceAvailability/logs/XRDFED_send.log new file mode 100644 index 0000000..db6168f --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/logs/XRDFED_send.log @@ -0,0 +1,2 @@ +Opening and sending documents +Done diff --git a/AAAOps/RedirectorServiceAvailability/send_metrics.py b/AAAOps/RedirectorServiceAvailability/send_metrics.py new file mode 100644 index 0000000..10e3260 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/send_metrics.py @@ -0,0 +1,83 @@ +#python3 + +# For a file where each line is a json valid document +# send those documents to metrics. +# Original script for AAAops + +import sys +import json +import requests +import traceback +import time +from datetime import datetime + +html_dir = '/var/www/html/aaa-probe/' +probes_json='KIBANA_PROBES_GENERAL.json' + +def update_dic_metrics(dic,metric_name,timestamp): + dic.update({'producer':'cmsaaa'}) + dic.update({'type':metric_name}) + dic.update({'timestamp':timestamp}) + + if dic.get('status')=='Available' and dic.get('version')!='': + dic.update({'codeStatus':1})#Available + + if dic.get('status')=='Unavailable' or dic.get('status')=='Degraded': + if dic.get('version')=='': + dic.update({'codeStatus':0})#Unavailable + else: + dic.update({'codeStatus':2})#Degraded + dic.update({'catCode':''}) + if dic.get('host')=='cms-xrd-global.cern.ch' or dic.get('host')=='cmsxrootd.fnal.gov' or dic.get('host')=='cms-xrd-transit.cern.ch': + dic.update({'catCode':0}) + + if dic.get('host')=='cms-xrd-global01.cern.ch' or dic.get('host')=='cms-xrd-global02.cern.ch': + dic.update({'catCode':1}) + + if dic.get('host')=='cmsxrootd2.fnal.gov' or dic.get('host')=='xrootd.unl.edu': + dic.update({'catCode':2}) + + #if dic.get('host')!='cms-xrd-global.cern.ch' or dic.get('host')!='cms-xrd-transit.cern.ch' or dic.get('host')!='cms-xrd-global01.cern.ch' or dic.get('host')!='cms-xrd-global02.cern.ch' or dic.get('host')!='cmsxrootd2.fnal.gov' or dic.get('host')!='xrootd.unl.edu': + #dic.update({'catCode':3}) + + if dic.get('host')=='vocms031.cern.ch' or dic.get('host')=='vocms032.cern.ch': + dic.update({'catCode':4}) + + if dic.get('catCode')=='': + dic.update({'catCode':3}) + + dic.update({'idb_fields':['codeStatus','status','Comment','version','service','host','xrdcp_below_time','xrdcp_above_time']}) + dic.update({'idb_tags':['status','host','codeStatus','catCode']}) + + +def send(document): + return requests.post('http://monit-metrics:10012/', data=json.dumps(document), headers={ "Content-Type": "application/json; charset=UTF-8"}) + +def send_and_check(document, should_fail=False): + response = send(document) + assert( (response.status_code in [200]) != should_fail), 'With document: {0}. Status code: {1}. Message: {2}'.format(document, response.status_code, response.text) + +def openjson_send(path,filename,metric_name,timestamp): + with open(path+filename) as jsonfile: + for line in jsonfile: + dic_line=(json.loads(line)) + update_dic_metrics(dic_line,metric_name,timestamp) + send_and_check(dic_line) + #print(dic_line) + +def main(): + metric_name="infrastructure" + now=datetime.now() + timestamp=int(datetime.timestamp(now))*1000 + + try: + print("Opening and sending documents") + openjson_send(html_dir,probes_json,metric_name,timestamp) + print("Done") + + except Exception: + print("Exception in checker procedure:") + traceback.print_exc() + sys.exit(2) +if __name__=="__main__": + sys.exit(main()) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh new file mode 100755 index 0000000..f6958b6 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -0,0 +1,36 @@ +#bin/sh + +notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch + +DATE=$(date) +echo $DATE +echo "ExecutingScript" + +cd /opt/TransferTeam/AAAOps/RedirectorServiceAvailability +[ -d logs ] || mkdir -p logs +logs=logs +thelog=$(pwd)/logs/uploadmetricGeneral.log + +# General script +date +echo INFO executing XRDFED-kibana-probe_JSON_General.py +python XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 +status=$? +echo INFO $logs/XRDFED_probe_json.log +cat $logs/XRDFED_probe_json.log +date +echo INFO running send_metrics.py +python3 send_metrics.py > $logs/XRDFED_send.log 2>&1 +echo INFO content of logs/XRDFED_send.log +cat $logs/XRDFED_send.log +date +echo INFO Done + +if [ -f $thelog ] ; then + a=1 + [ $status -eq 0 ] && { grep -q -i "caught overall timeout" $logs/XRDFED_probe_json.log ; [ $(expr $a + $?) -eq $a ] && status=1 ; } ; + if [ $status -ne 0 ] ; then + printf "$(/bin/hostname) $(basename $0)\n$(date)\n$(ls -al $thelog )\n$(cat $thelog)\n" | mail -s "$(/bin/hostname) $(basename $thelog)" $(echo $notifytowhom | sed 's#NOSPAMPLEASE##') -a $thelog + fi +fi + From 27dbc6ca4e92c3bfbda4af8d602931eb8d7f5703 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 25 Aug 2021 13:47:12 -0400 Subject: [PATCH 064/192] Initial commit for aaa subscription scripts --- AAAOps/FedProbeSendAAAMetrics/README | 37 + .../FedProbeSendAAAMetrics/aaa_federation.py | 148 ++ .../check_subscribed_sites.sh | 7 + .../FedProbeSendAAAMetrics/check_timeout.sh | 18 + .../check_upstream_redirector_examples.sh | 6 + .../check_vofeed_from_xrdmapc.sh | 64 + .../check_xrdmapc_from_vofeed.sh | 112 ++ .../compare_timeout_list.sh | 8 + .../FedProbeSendAAAMetrics/create_fedmaps.py | 1382 +++++++++++++++++ AAAOps/FedProbeSendAAAMetrics/listSrv.sh | 5 + .../FedProbeSendAAAMetrics/mapHostSitename.sh | 5 + .../probe_create_send_aaa_metrics.sh | 54 + 12 files changed, 1846 insertions(+) create mode 100644 AAAOps/FedProbeSendAAAMetrics/README create mode 100755 AAAOps/FedProbeSendAAAMetrics/aaa_federation.py create mode 100755 AAAOps/FedProbeSendAAAMetrics/check_subscribed_sites.sh create mode 100755 AAAOps/FedProbeSendAAAMetrics/check_timeout.sh create mode 100644 AAAOps/FedProbeSendAAAMetrics/check_upstream_redirector_examples.sh create mode 100755 AAAOps/FedProbeSendAAAMetrics/check_vofeed_from_xrdmapc.sh create mode 100755 AAAOps/FedProbeSendAAAMetrics/check_xrdmapc_from_vofeed.sh create mode 100755 AAAOps/FedProbeSendAAAMetrics/compare_timeout_list.sh create mode 100755 AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py create mode 100755 AAAOps/FedProbeSendAAAMetrics/listSrv.sh create mode 100755 AAAOps/FedProbeSendAAAMetrics/mapHostSitename.sh create mode 100755 AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh diff --git a/AAAOps/FedProbeSendAAAMetrics/README b/AAAOps/FedProbeSendAAAMetrics/README new file mode 100644 index 0000000..d8bb766 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/README @@ -0,0 +1,37 @@ +[1] Essential Files and Directories +create_fedmaps.py +aaa_federation.py +probe_create_send_aaa_metrics.sh +CMSMonitoring +credentials.json + +[2] Needed Directories Created +out +logs + +[3] Utility Scripts +check_timeout.sh +compare_timeout_list.sh +mapHostSitename.sh +listSrv.sh +check_xrdmapc_from_vofeed.sh +check_vofeed_from_xrdmapc.sh +check_upstream_redirector_examples.sh +check_production_sites.sh + +[4] Installing +# 1 +cd /opt +git clone git@github.com:CMSCompOps/TransferTeam.git +cd TransferTeam/AAAOps/FedProbeSendAAAMetrics +cp ~/credentials.json ./ +# OR +# 2 +cd /opt/TransferTeam +git pull origin master +cd AAAOps/FedProbeSendAAAMetrics +cp ~/credentials.json ./ +git clone https://github.com/dmwm/CMSMonitoring.git +crontab -l > crontab +echo '14,29,44,59 * * * * /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh > /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/logs/probe_create_send_aaa_metrics.log 2>&1' >> crontab +# Add the cron to the gitlab diff --git a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py new file mode 100755 index 0000000..90079d0 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python + +import os +import sys +import subprocess +import urllib +import json +from datetime import datetime +import collections +import stomp +import argparse +from CMSMonitoring.StompAMQ import StompAMQ +import uuid +from itertools import islice +import hashlib +import stomp +from uuid import uuid4 +from urllib.request import urlopen +import time + +#print("stomp version", stomp.__version__) + +FedProbeSendAAAMetrics = '/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' +FedProbeSendAAAMetrics = '/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' + +class OptionParser(): + def __init__(self): + "User based option parser" + self.parser = argparse.ArgumentParser() + msg = "Send aaa subscription json file via StompAMQ to a broker, provide broker credentials in JSON file" + self.parser.add_argument("--amq", action="store", + dest="amq", default="", help=msg, required=True) + #self.parser.add_argument("--create_json_only", action="store", + # dest="create_json_only", default="", help=msg, required=False) + + +class MyListener(stomp.ConnectionListener): + def on_error(self, headers, message): + print('received an error "%s"' % message) + + def on_message(self, headers, message): + print('received a message "%s"' % message) + + +def credentials(fname=None): + "Read credentials from MONIT_BROKER environment" + if not fname: + fname = os.environ.get('MONIT_BROKER', '') + if not os.path.isfile(fname): + raise Exception( + "Unable to locate MONIT credentials, please setup MONIT_BROKER") + return {} + with open(fname, 'r') as istream: + data = json.load(istream) + return data + +def parseFederationJson(): + ''' + modifies federation.json with tags and + Return json object to feed cern metris + ''' + #url = 'http://vocms039.cern.ch/aaa-fedinfo/federations.json' + #result = json.load(urlopen(url)) + #print ("open f") + with open(FedProbeSendAAAMetrics+'out/federations.json') as f: url = f.read() + #print ("url ",url) + result = json.loads(url) + + if result['prod'] and result['trans']: + fed_data = createFlattenJson(result) + #print (opts.create_json_only) + f = open(FedProbeSendAAAMetrics+"fed.json", "r") + #if opts.create_json_only : + # print (" 2 ",opts.create_json_only) + # for x in f: + # data = json.loads(x) + #else: + for x in f: + data = json.loads(x) + #print ( data ) + yield data + +def createFlattenJson(json_file): + ''' + flattens the federation.json file + ''' + #print ("createFlattenJson") + for k,v in json_file.items(): + #print ( k, v.keys() ) + #for dic in v: + # print ( k, dic.keys() ) + #break + if ("prod" in k): + statusCode = 1 + stateName = "Production Federation" + elif ("trans" in k): + statusCode = 2 + stateName = "Transitional Federation" + else: + statusCode = 3 + stateName = "not at any federation" + for site in v: + #print ( k, site ) + #my_json = ('{"siteName" : "%s", "state" : "%s", "statusCode" : %s, "stateName" : "%s"}' %(site, k, statusCode, stateName)) + #for isite in range(len(site['sites'])): + # if 'XROOTD' not in site['flavors'][isite] : continue + # #contact = site['contact'][0] + # ##print (site['sites'][isite], " contact len ",len(site['contact'])) + # #if not contact : contact = site['contact'][(len(site['contact']) - 1)] + # #print (" s = ",site['sites'][isite], site['flavors'][isite], site['endpoints'][isite], site['federation'][isite]) #print (" site = ",site['sites']) + my_json = ('{"siteName" : "%s", "state" : "%s", "statusCode" : %s, "stateName" : "%s", "flavor" : "%s", "endpoint" : "%s", "xrootd_version" : "%s", "xrootd_role" : "%s", "storage" : "%s", "contact" : "%s"}' %(site['sites'], k, statusCode, stateName, site['flavors'], site['endpoints'], site['xrootd_version'], site['xrootd_role'], site['xrootd_storage'], site['contact'])) + with open(FedProbeSendAAAMetrics+"fed.json","a") as f: + f.write("%s\n" %my_json) + +if __name__ == "__main__": + optmgr = OptionParser() + opts = optmgr.parser.parse_args() + payload = parseFederationJson() + #for ipay in payload: #bockjoo comment this out when in production + # print (ipay) #bockjoo comment this out when in production + ##print ("Check "+FedProbeSendAAAMetrics+"fed.json") + #sys.exit(0) # bockjoo to uncomment + if opts.amq: + creds = credentials(opts.amq) + username = creds['username'] + password = creds['password'] + topic = creds['topic'] + host, port = creds['host_and_ports'].split(':') + port = int(port) + producer = creds['producer'] + topic = creds['topic'] + hosts = [(host, port)] + if creds and StompAMQ: + amq = StompAMQ(username, password, producer, topic, key=None, + cert=None, validation_schema=None, host_and_ports=[(host, port)]) + eod = False + wait_seconds = 10 + while not eod: + messages = [] + for d in payload: + notif,_,_ = amq.make_notification(d, "aaa_federations_document", dataSubfield=None) + messages.append(notif) + if messages: + print(messages) + amq.send(messages) + time.sleep(wait_seconds) + else: + eod = True diff --git a/AAAOps/FedProbeSendAAAMetrics/check_subscribed_sites.sh b/AAAOps/FedProbeSendAAAMetrics/check_subscribed_sites.sh new file mode 100755 index 0000000..4f799bc --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_subscribed_sites.sh @@ -0,0 +1,7 @@ +#!/bin/bash +what=Production +[ $# -gt 0 ] && what=$1 +sites=$(grep "$what" /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/fed.json | awk '{print $3}' | sort -u) +printf "$sites\n" +echo "Count: $(echo $sites | wc -w)" + diff --git a/AAAOps/FedProbeSendAAAMetrics/check_timeout.sh b/AAAOps/FedProbeSendAAAMetrics/check_timeout.sh new file mode 100755 index 0000000..c3a676e --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_timeout.sh @@ -0,0 +1,18 @@ +#!/bin/bash +endpoints=$(grep ^T[0-9]_ create_fedmaps.log | awk '{print $2}' | sort -u) +for e in $endpoints ; do + #echo $e + log=tmp/$(echo $e | sed 's#:##g').log + xrdfs $e query config version > $log 2>&1 & + theps=$! + i=0 + while [ $i -lt 30 ] ; do + + ps auxww | grep -v grep | grep $theps | awk '{print "+"$2"+"}' | grep -q "+"$theps"+" + + [ $? -ne 0 ] && break + i=$(expr $i + 1) + sleep 1 + done + echo $i $e $(cat $log) +done diff --git a/AAAOps/FedProbeSendAAAMetrics/check_upstream_redirector_examples.sh b/AAAOps/FedProbeSendAAAMetrics/check_upstream_redirector_examples.sh new file mode 100644 index 0000000..b78d0a0 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_upstream_redirector_examples.sh @@ -0,0 +1,6 @@ +#!/bin/bash +n=1000 ; xrdmapc --list all cms-xrd-global01.cern.ch:1094 2>/dev/null | grep -A $n ^1 | grep -B $n kr | grep "^1\|cms-t2-se01.sdfarm.kr" +n=1000 ; xrdmapc --list all cms-xrd-global.cern.ch:1094 2>/dev/null | grep -A $n ^1 | grep -B $n xrootd-cms-01.cr.cnaf.infn.it:0 | grep "^1\|xrootd-cms-01.cr.cnaf.infn.it:0" +n=1000 ; xrdmapc --list all cms-xrd-global.cern.ch:1094 2>/dev/null | grep -A $n ^2 | grep -B $n xrootd-cms-01.cr.cnaf.infn.it:0 | grep "^2\|xrootd-cms-01.cr.cnaf.infn.it:0" +n=1000 ; xrdmapc --list all cms-xrd-global.cern.ch:1094 2>/dev/null | grep -A $n ^2 | grep -B $n xrootd-cms-01.cr.cnaf.infn.it:0 | grep -B 1 "^2\|xrootd-cms-01.cr.cnaf.infn.it:0" + diff --git a/AAAOps/FedProbeSendAAAMetrics/check_vofeed_from_xrdmapc.sh b/AAAOps/FedProbeSendAAAMetrics/check_vofeed_from_xrdmapc.sh new file mode 100755 index 0000000..577aba5 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_vofeed_from_xrdmapc.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Checks all xrootd endpoint in vofeed https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml is found from xrdmapcall +# +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +#xredirs=$(cat $FedProbeSendAAAMetrics/fed.json | cut -d, -f6 | sort -u | cut -d\" -f4 | cut -d: -f1) +#xredirs=$(grep " Man " $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt | awk '{print $NF}' | cut -d: -f1 | sort -u) +xredirs=$(grep " Man " $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt | cut -d: -f2- | awk '{print $3}' | sort -u) +output=$(printf "%5s %60s %5s\n" Found Xredirtor MLevel) +for xredir in $xredirs ; do + MLevel=$(echo $(grep $xredir $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt | cut -d: -f2- | awk '{print $1}' | sort -u) | sed 's# #+#g') + Found=$(grep -q $xredir $FedProbeSendAAAMetrics/fed.json ; echo $?) + output="$output\n"$(printf "%5s %60s %5s\n" $Found $xredir $MLevel) + # + # Analyze the outcome + # + # 1 Found=1 MLevel!=1 -> 1 $xredir can be one of Found=1 MLevel=1 + # 2 $xredir could be found from the alias in vofeed + +done +printf "$output\n" +exit 0 + +Found Xredirtor MLevel + 0 cms-aaa-manager01.gridpp.rl.ac.uk:1094 2+3 + 0 cmsio7.rc.ufl.edu:1094 2 + 1 cmsxrootd2.fnal.gov:1094 1 + 1 cmsxrootd.ihep.ac.cn:1094 2 + 0 cmsxrootd-site1.fnal.gov:1094 2 + 0 cmsxrootd-site2.fnal.gov:1094 2 + 0 cmsxrootd-site3.fnal.gov:1094 2 + 1 dmsdcatst03.fnal.gov:1094 2 + 1 llrxrd-redir.in2p3.fr:1094 1 + 1 llrxrd-redir.in2p3.fr:1194 2 + 1 llrxrd-redir.in2p3.fr:1294 2 + 0 osg-se.sprace.org.br:1094 2 + 0 pubxrootd.hep.wisc.edu:1094 2 + 0 redirector.t2.ucsd.edu:1094 2 + 0 xroot02.ncg.ingrid.pt:1094 2 + 1 xrootd01.grid.hep.ph.ic.ac.uk:1094 1 + 1 xrootd.ba.infn.it:1094 1 + 1 xrootd.ba.infn.it:1194 2 + 1 xrootd.ba.infn.it:1294 2 + 1 xrootd-cms-01.cr.cnaf.infn.it:0 3 + 0 xrootd-cms-01.cr.cnaf.infn.it:1094 3 + 0 xrootd-cms-02.cr.cnaf.infn.it:1094 3 + 0 xrootd.cmsaf.mit.edu:1094 2 + 1 xrootd-cms-redir-int.cr.cnaf.infn.it:1094 1 + 1 xrootd-cms-redir-int.cr.cnaf.infn.it:1194 2 + 1 xrootd-cms-redir-int.cr.cnaf.infn.it:1294 2 + 1 xrootd-cms-uk.gridpp.rl.ac.uk:1094 2 https://ggus.eu/index.php?mode=ticket_info&ticket_id=153496&come_from=submit + 1 xrootd-es-cie.ciemat.es:1096 2 + 1 xrootd-es-pic.pic.es:1096 2 + 0 xrootd.hepgrid.uerj.br:1094 2 + 0 xrootd.hep.kbfi.ee:1094 2 + 0 xrootd-local.unl.edu:1094 2 + 0 xrootd.rcac.purdue.edu:1094 2 + 1 xrootd-redic.pi.infn.it:1094 1 + 1 xrootd-redic.pi.infn.it:1194 2 + 1 xrootd-redic.pi.infn.it:1294 2 + 1 xrootd-redir1-vanderbilt.sites.opensciencegrid.org:1094 2 + 1 xrootd-redir2-vanderbilt.sites.opensciencegrid.org:1094 2 + 0 xrootd-redir.ultralight.org:1094 2 diff --git a/AAAOps/FedProbeSendAAAMetrics/check_xrdmapc_from_vofeed.sh b/AAAOps/FedProbeSendAAAMetrics/check_xrdmapc_from_vofeed.sh new file mode 100755 index 0000000..fa44229 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_xrdmapc_from_vofeed.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# +# Checks all xrootd endpoint in vofeed https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml is found from xrdmapcall +# +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +xredirs=$(cat $FedProbeSendAAAMetrics/fed.json | cut -d, -f6 | sort -u | cut -d\" -f4 | cut -d: -f1) +output=$(printf "%5s %50s %3s\n" Found Xredirtor Tier) +for xredir in $xredirs ; do + Tier=$(echo $(grep $xredir $FedProbeSendAAAMetrics/fed.json | cut -d\" -f4 | cut -d_ -f1 | sed 's#T##g' | sort -u) | sed 's# #+#g') + Found=$(grep -q $xredir $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt ; echo $?) + output="$output\n"$(printf "%5s %50s %3s\n" $Found $xredir $Tier) +done +printf "$output\n" +exit 0 +Found Xredirtor Tier + 1 bonner04.rice.edu 3 + 1 brux11.hep.brown.edu 3 + 0 ccsrm.ihep.ac.cn 2 + 1 ccxrdcms.in2p3.fr 1 + 0 ceph-gw10.gridpp.rl.ac.uk 1 + 0 ceph-gw11.gridpp.rl.ac.uk 1 + 1 cluster142.knu.ac.kr 3 + 0 cms03.lcg.cscs.ch 2 + 0 cms-aaa-manager01.gridpp.rl.ac.uk 1 + 0 cmsdata.phys.cmu.edu 3 + 0 cmsio7.rc.ufl.edu 2 + 0 cmsrm-xrootd01.roma1.infn.it 2 + 0 cmsrm-xrootd02.roma1.infn.it 2 + 0 cms-se0.kipt.kharkov.ua 2 + 1 cms-se.hep.uprm.edu 3 + 0 cms-t2-se01.sdfarm.kr 2 + 1 cmsxrd.ts.infn.it 3 + 1 cmsxrootd-redirectors.gridka.de 1 + 0 cmsxrootd-site1.fnal.gov 1 + 0 cmsxrootd-site2.fnal.gov 1 + 0 cmsxrootd-site3.fnal.gov 1 + 0 dc2-grid-64.brunel.ac.uk 2 + 1 dcache-cms-xrootd.desy.de 2 + 1 eos.grid.vbc.ac.at 2 + 0 eymir.grid.metu.edu.tr 2 + 0 gaexroot01.ciemat.es 2 + 0 gfe02.grid.hep.ph.ic.ac.uk 2 + 0 grid02.physics.uoi.gr 2 + 0 grid143.kfki.hu 2 + 1 grid71.phy.ncu.edu.tw 3 + 0 grid-dcache.physik.rwth-aachen.de 2 + 0 grse001.inr.troitsk.ru 2 + 0 hask.csc.fi 2 + 1 hepcms-0.umd.edu 3 + 0 heplnx228.pp.rl.ac.uk 2 + 0 heplnx229.pp.rl.ac.uk 2 + 1 hepxrd01.colorado.edu 3 + 1 ingrid-se03.cism.ucl.ac.be 2 + 0 ingrid-se04.cism.ucl.ac.be 2 + 1 ingrid-se05.cism.ucl.ac.be 2 + 0 ingrid-se06.cism.ucl.ac.be 2 + 1 kodiak-se.baylor.edu 3 + 0 lcgse01.phy.bris.ac.uk 2 + 0 lcgsexrd.jinr.ru 2 + 1 llrpp01.in2p3.fr 2 + 1 lyoeos.in2p3.fr 3 + 1 lyogrid06.in2p3.fr 3 + 0 maite.iihe.ac.be 2 + 0 node12.datagrid.cea.fr 2 + 0 nute.csc.fi 2 + 0 osg-se.sprace.org.br 2 + 1 pcncp22.ncp.edu.pk 2 + 0 polgrid4.in2p3.fr 2 + 0 pool01.ifca.es 2 + 0 pool02.ifca.es 2 + 0 pool03.ifca.es 2 + 0 pool04.ifca.es 2 + 0 pool05.ifca.es 2 + 0 pool06.ifca.es 2 + 0 pool07.ifca.es 2 + 0 pool08.ifca.es 2 + 0 pubxrootd.hep.wisc.edu 2 + 0 redirector.t2.ucsd.edu 2 + 0 sbgse1.in2p3.fr 2 + 0 se01.grid.nchc.org.tw 2 + 0 se3.itep.ru 2 + 0 se.cis.gov.pl 2 + 0 se-xrd01.jinr-t1.ru 1 + 0 ss-01.recas.ba.infn.it 2 + 0 ss-02.recas.ba.infn.it 2 + 0 ss-03.recas.ba.infn.it 2 + 0 stormgf1.pi.infn.it 2 + 1 stormgf2.pi.infn.it 2 + 1 stormgf3.pi.infn.it 2 + 1 storm.mib.infn.it 3 + 0 t2-cms-xrootd01.desy.de 2 + 0 t2-cms-xrootd02.desy.de 2 + 0 t2-cms-xrootd03.desy.de 2 + 0 t2-xrdcms.lnl.infn.it 2 + 1 t3se01.psi.ch 3 + 0 vobox0002.m45.ihep.su 2 + 0 xroot02.ncg.ingrid.pt 2 + 0 xrootd01-cmst1.pic.es 1 + 0 xrootd02-cmst1.pic.es 1 + 0 xrootd-cms-01.cr.cnaf.infn.it 1 + 0 xrootd-cms-02.cr.cnaf.infn.it 1 + 0 xrootd.cmsaf.mit.edu 2 + 0 xrootd-es-cie.ciemat.es 2 + 0 xrootd.hepgrid.uerj.br 2 + 0 xrootd.hep.kbfi.ee 2 + 0 xrootd-local.unl.edu 2 + 0 xrootd.rcac.purdue.edu 2 + 0 xrootd-redir.ultralight.org 2 + 1 xrootd-vanderbilt.sites.opensciencegrid.org 2 + 1 xroot.pp.rl.ac.uk 2 + diff --git a/AAAOps/FedProbeSendAAAMetrics/compare_timeout_list.sh b/AAAOps/FedProbeSendAAAMetrics/compare_timeout_list.sh new file mode 100755 index 0000000..63a0433 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/compare_timeout_list.sh @@ -0,0 +1,8 @@ +#!/bin/bash +list1=$(grep ^T[0-9]_ create_fedmaps.log | awk '{print $1}' | sort -u) +list2=$(cat aaa_federation.log | sed "s#'siteName'#\nsiteName'#g" | grep ^siteName | awk '{print $2" "$13" "$15" "$17}' | grep timeout | cut -d\' -f2 | sort -u) + +echo "$(echo $list1)" +echo "$(echo $list2)" + +exit 0 diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py new file mode 100755 index 0000000..32f277c --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -0,0 +1,1382 @@ +try: import xml.etree.ElementTree as ET +except ImportError: from elementtree import ElementTree as ET +try: import json +except ImportError: import simplejson as json + +import sys +#import urllib2, httplib, sys +#import httplib, sys + +try : import urllib +except : import urllib2 + +try: + # For Python 3.0 and later + import urllib.request +except ImportError: + # Fall back to Python 2's urllib2 + import urllib2 + +# from XRDFED-kibana-probe.py +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +#import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import socket +#import path +#import os.path +from os import path +import numpy as np + +# global vars: sites, output +# cmsTopology : from vofeed, +# sites : from vofeed and updated as we proceed +# output : {"prod" : [...], "trans" : [...], "nowhere" : [...]} + +THEPATH='/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' +#VOFEED='http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii' +VOFEED='https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml' +LOCKFILE=THEPATH+'create_fedmaps.lock' +timeout_sec = 10 * 60 + +output = {"prod" : [], "trans" : [], "nowhere" : []} +sites = {} +xrdmapc_output_prod = '' +xrdmapc_output_tran = '' +gRedirectors=['cms-xrd-global01.cern.ch', 'cms-xrd-global02.cern.ch'] +tRedirectors=['cms-xrd-transit.cern.ch'] +topRedirectors = [gRedirectors[0],]+tRedirectors + +if not path.exists(THEPATH+'mapHostSitenames.py') : + with open(THEPATH+'mapHostSitenames.py','w') as data : data.write('HostSitenames = {}') + +if not path.exists(THEPATH+'siteStoages.py') : + with open(THEPATH+'siteStoages.py','w') as data : data.write('SiteStorages = {}') + +if not path.exists(THEPATH+'DNSARecords.py') : + with open(THEPATH+'DNSARecords.py','w') as data : data.write('DNSARecords = {}') + +if not path.exists(THEPATH+'xrdVersions.py') : + with open(THEPATH+'xrdVersions.py','w') as data : data.write('XrdVersions = {}') + +from mapHostSitenames import HostSitenames +from siteStoages import SiteStorages +from DNSARecords import DNSARecords +from xrdVersions import XrdVersions + +class Alarm(Exception): + pass + +def alarm_handler(signum, frame): + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + clear_lock() + sys.exit(2) + raise Alarm + +def clear_lock(): + try: + os.unlink(LOCKFILE) + except (Exception,e) : + print ("could not remove lockfile:"+str(e)) + +def xrd_info(redirector,what): + theargs = [redirector, "query", "config", what] + if 'version' in what or 'role' in what : command = 'xrdfs' + else : + command = 'xrdmapc' + theargs = [ '--list', 'all', redirector ] + config_out = "unknown" + count = 0 + count_limit = 3 + timelimit = 2 + if 'xrdmapc' in command : + timelimit = 90 + count_limit = 1 + while (count < count_limit): + (errtext,out,err,elapsed) = run_xrd_commands(command, theargs, timelimit) #"xrdfs", + #[redirector, + # "query","config", # 1:kXR_QStats + # what]) # a_ll stats + + if 'xrdmapc' in command : + if out : + errtext = err + config_out = out + break + if err: + #errtext = '' + #config_out = 'error' + config_out = err.replace(b'\n',b'').decode() + else: + #config_out=out + #if 'xrdmapc' in command : + # break + #else : + config_out = out.replace(b'\n',b'').decode() + # # to check if count matters config_out=config_out+'+'+str(count) + if out.replace(b'\n',b'') : break + time.sleep(1) + count += 1 + if not config_out : config_out = 'timeout' + #print ( 'DEBUG count = ',count,' config_out ',config_out, ' redirector ',redirector) + return (errtext,config_out,out) + +def run_xrd_commands(cmd,args,timelimit): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + #xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + # "-DIConnectTimeout","30", + # "-DITransactionTimeout","60", + # "-DIRequestTimeout","60" ] + args + xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + #xrd_args = [ 'perl','-e',"alarm 30 ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + ] + args + #if 'xrdmapc' in cmd : + # xrd_args = [ cmd, ] + args + try: + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + ret = proc.returncode + elapsed = (time.time() - start) + #print ( ' out ',out ) + #if 'xrdmapc' in cmd : return ('',out,err,elapsed) + err_redir_index = err.rfind(b'Received redirection to') + err_index3010 = err.rfind(b'(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind(b'(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + errtxt = '' + else: + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind(b'Last server error') + if err_index >= 0: + err_end_index=err.find(b"\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except (Exception,e) : + errtext = errtxt + "Exception: "+str(e) + out = 'Try did not work :O' + print(out) + dev_null.close() + + return (errtxt,out,err,elapsed) + + +def getDataFromURL(url, header = {}): + try: + # For Python 3.0 and later + request = urllib.request.Request(url, headers=header) + urlObj = urllib.request.urlopen(request) + data = urlObj.read() + except ImportError: + request = urllib2.Request(url, headers=header) + urlObj = urllib2.urlopen(request) + data = urlObj.read() + return data + +# Hard-coded one +def getGlobalRedirectors(): + + siteName='T0_CH_CERN' + #gRedirectors=['cms-xrd-global01.cern.ch', 'cms-xrd-global02.cern.ch'] + #tRedirectors=['cms-xrd-transit.cern.ch'] + ret = {} + #ret[siteName] = {} + ret['sites'] = [] + ret['hosts'] = [] + ret['flavors'] = [] + ret['endpoints'] = [] + ret['name'] = [] + ret['contact'] = [] + ret['xrootd_version'] = [] + ret['xrootd_role'] = [] + ret['xrootd_servers'] = [] + ret['xrootd_storage'] = [] + if SiteStorages[ siteName ] : xrootd_storage = SiteStorages[ siteName ] + else : + xrootd_storage = getStorageFromStorageJson ( siteName ) + updateSiteStorage ( siteName , xrootd_storage ) + ret['federation'] = [] + + # Production + for gRedir in gRedirectors: + ret['sites'].append(siteName) + ret['hosts'].append(gRedir) + ret['flavors'].append('XROOTD') + ret['endpoints'].append(gRedir+':1094') + ret['name'].append('CERN-PROD') + ret['contact'].append(siteName) + #(err_info,xrootd_version,dump_info) = xrd_info(gRedir+':1094',"version") + #(err_info,xrootd_role,dump_info) = xrd_info(gRedir+':1094',"role") + ret['xrootd_version'].append('') + ret['xrootd_role'].append('') + ret['xrootd_servers'].append('_^_') + ret['xrootd_storage'].append(xrootd_storage) + ret['federation'].append('prod') + + # Transitional + for tRedir in tRedirectors: + ret['sites'].append(siteName) + ret['hosts'].append(tRedir) + ret['flavors'].append('XROOTD') + ret['endpoints'].append(tRedir+':1094') + ret['name'].append('CERN-PROD') + ret['contact'].append(siteName) + #(err_info,xrootd_version,dump_info) = xrd_info(gRedir+':1094',"version") + #(err_info,xrootd_role,dump_info) = xrd_info(gRedir+':1094',"role") + ret['xrootd_version'].append('') + ret['xrootd_role'].append('') + ret['xrootd_servers'].append('_^_') + ret['xrootd_storage'].append(xrootd_storage) + ret['federation'].append('trans') + return ret + +def getSites(): + XML = getDataFromURL(VOFEED) + XML = ET.fromstring(XML) + #sites = XML.findall('atp_site') + sites = XML.findall('.//atp_site') + ret = {} + for site in sites: + + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + services = site.findall('.//service') + ret[siteName] = {} + ret[siteName]['sites'] = [] + ret[siteName]['hosts'] = [] + ret[siteName]['flavors'] = [] + ret[siteName]['endpoints'] = [] + ret[siteName]['name'] = [] + ret[siteName]['contact'] = [] + ret[siteName]['xrootd_version'] = [] + ret[siteName]['xrootd_role'] = [] + ret[siteName]['xrootd_servers'] = [] + ret[siteName]['xrootd_storage'] = [] + ret[siteName]['federation'] = [] + #print ('getSites() ret keys ',ret.keys() ) + #print ('getSites() ret["T0_CH_CERN"] ',ret["T0_CH_CERN"]) + name = '' + contact = '' + for site in sites: + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + #print ('getSites() site ',siteName) + services = site.findall('.//service') + #ret[siteName]['name'].append(site.attrib['name']) + if site.attrib['name'] : name = site.attrib['name'] + contact = None + try: + #ret[siteName]['contact'].append(site.attrib['contact']) + contact = site.attrib['contact'] + except: + #ret[siteName]['contact'].append("") + contact = siteName + #if site.attrib['contact'] : contact = site.attrib['contact'] + + #print ('getSites() site contact ',) + for service in services: + flavor = service.attrib['flavour'] + try: + endpoint=service.attrib['endpoint'] + except: + endpoint=service.attrib['flavour'] + + xrootd_version='__' + xrootd_role='__' + #if flavor in 'XROOTD': + # (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + # (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + serviceName = service.attrib['hostname'] + + #print ('getSites() services siteName ',siteName) + ret[siteName]['sites'].append(siteName) + ret[siteName]['hosts'].append(serviceName) + ret[siteName]['flavors'].append(flavor) + ret[siteName]['endpoints'].append(endpoint) + ret[siteName]['xrootd_version'].append(xrootd_version) + ret[siteName]['xrootd_role'].append(xrootd_role) + ret[siteName]['xrootd_servers'].append('_^_') + #print ('getSites() getStorageFromStorageJson siteName ',siteName) + xrootd_storage = getStorageFromStorageJson ( siteName) + #print ('getSites() xrootd_storage ',xrootd_storage) + #if siteName in SiteStorages : print (' found it ') + #else : print ('not found ') + if siteName in SiteStorages : + #print ( 'A ') + xrootd_storage = SiteStorages[ siteName ] + else : + #print ( 'B ') + xrootd_storage = getStorageFromStorageJson ( siteName ) + updateSiteStorage ( siteName , xrootd_storage ) + #print ('getSites() done sitestorage ') + ret[siteName]['xrootd_storage'].append(xrootd_storage) + ret[siteName]['name'].append(name) + ret[siteName]['contact'].append(contact) + ret[siteName]['federation'].append('nowhere') + #print ('getSites() done ') + return ret + + +def getStorageFromStorageJson ( site ): + import os.path + from os import path + if not path.exists('/cvmfs/cms.cern.ch/SITECONF/'+site+'/storage.json') : return 'no_storage_json' + with open('/cvmfs/cms.cern.ch/SITECONF/'+site+'/storage.json') as f: storage = f.read() + storage = json.loads(str(storage)) + xrootd_volume = 'info_missing' + for vol in storage : + volume = str(vol['volume']) # ' '.join([str(elem) for elem in str(vol['volume']).split('_')]) + if 'FEDERATION' in volume.upper() : continue + for proto in vol['protocols'] : + #print ( 'storage volume ',volume,proto['protocol'] ) + if 'XRootD' in str(proto['protocol']) : xrootd_volume = volume + if 'info_missing' in xrootd_volume : + for vol in storage : + volume = str(vol['volume']) # ' '.join([str(elem) for elem in str(vol['volume']).split('_')]) + if 'FEDERATION' in volume.upper() : continue + for proto in vol['protocols'] : + #print ( 'storage volume ',volume,proto['protocol'] ) + if 'WebDAV' in str(proto['protocol']) : xrootd_volume = volume + #print ( site, xrootd_volume ) + # 3 ( + #print ( 'storage volume ', len(storage), (str(storage[0]['volume'])).split('_')[1:]) + #print ( 'storage volume ', len(storage), storage[1]) + #print ( 'storage volume ', len(storage), storage[2]) + return xrootd_volume + +def getStorages(): + XML = getDataFromURL(VOFEED) + XML = ET.fromstring(XML) + #sites = XML.findall('atp_site') + sites = XML.findall('.//atp_site') + ret = {} + for site in sites: + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + #services = site.findall('.//service') + ret[siteName] = {} + ret[siteName]['sites'] = [] + ret[siteName]['hosts'] = [] + ret[siteName]['flavors'] = [] + ret[siteName]['endpoints'] = [] + ret[siteName]['name'] = [] + ret[siteName]['contact'] = [] + ret[siteName]['xrootd_version'] = [] + ret[siteName]['xrootd_role'] = [] + ret[siteName]['xrootd_servers'] = [] + ret[siteName]['xrootd_storage'] = [] + name = '' + contact = '' + storage_guessed = '' + for site in sites: + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + se_resources = site.findall('.//se_resource') + storage_guessed = 'NI' + for i in se_resources : + if i.attrib['id'] == 'RD3PCP': + try : + path = se_resources[1].attrib['path'] + #print (siteName, path ) + if 'HADOOP' in path.upper() : storage_guessed = 'hadoop' + if 'DCACHE' in path.upper() : storage_guessed = 'dcache' + if '/PNFS/' in path.upper() : storage_guessed = 'dcache' + if 'CEPH' in path.upper() : storage_guessed = 'ceph' + if 'LUSTRE' in path.upper() : storage_guessed = 'lustre' + if 'STORM' in path.upper() : storage_guessed = 'posix' + if 'GPFS' in path.upper() : storage_guessed = 'gpfs' + if 'EOS' in path.upper() : storage_guessed = 'eos' + except : + storage_guessed = 'NI' + if i.attrib['id'] == 'WRDEL3PCP' : + try : + path = se_resources[1].attrib['path'] + #print (siteName, path ) + if 'HADOOP' in path.upper() : storage_guessed = 'hadoop' + if 'DCACHE' in path.upper() : storage_guessed = 'dcache' + if '/PNFS/' in path.upper() : storage_guessed = 'dcache' + if 'CEPH' in path.upper() : storage_guessed = 'ceph' + if 'LUSTRE' in path.upper() : storage_guessed = 'lustre' + if 'STORM' in path.upper() : storage_guessed = 'posix' + if 'GPFS' in path.upper() : storage_guessed = 'gpfs' + if 'EOS' in path.upper() : storage_guessed = 'eos' + except : + storage_guessed = 'NI' + + services = site.findall('.//service') + #ret[siteName]['name'].append(site.attrib['name']) + if site.attrib['name'] : name = site.attrib['name'] + try: + #ret[siteName]['contact'].append(site.attrib['contact']) + contact = site.attrib['contact'] + except: + #ret[siteName]['contact'].append("") + contact = siteName + #if site.attrib['contact'] : contact = site.attrib['contact'] + + for service in services: + flavor = service.attrib['flavour'] + try: + endpoint=service.attrib['endpoint'] + except: + endpoint=service.attrib['flavour'] + + xrootd_version='__' + xrootd_role='__' + #if flavor in 'XROOTD': + # (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + # (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + serviceName = service.attrib['hostname'] + if 'XROOTD' in flavor : + #print ( siteName, serviceName ) + if 'HADOOP' in serviceName.upper() : storage_guessed = 'hadoop' + if 'DCACHE' in serviceName.upper() : storage_guessed = 'dcache' + if '/PNFS/' in serviceName.upper() : storage_guessed = 'dcache' + if 'CEPH' in serviceName.upper() : + storage_guessed = 'ceph' + #print (siteName, serviceName, 'ceph') + if 'LUSTRE' in serviceName.upper() : storage_guessed = 'lustre' + if 'STORM' in serviceName.upper() : storage_guessed = 'posix' + if 'GPFS' in serviceName.upper() : storage_guessed = 'gpfs' + if 'EOS' in serviceName.upper() : storage_guessed = 'eos' + + ret[siteName]['sites'].append(siteName) + ret[siteName]['hosts'].append(serviceName) + ret[siteName]['flavors'].append(flavor) + ret[siteName]['endpoints'].append(endpoint) + ret[siteName]['xrootd_version'].append(xrootd_version) + ret[siteName]['xrootd_role'].append(xrootd_role) + ret[siteName]['xrootd_servers'].append('_^_') + ret[siteName]['xrootd_storage'].append( siteName+'+'+storage_guessed ) + #ret[siteName]['xrootd_storage_guessed'].append( storage_guessed ) + ret[siteName]['name'].append(name) + ret[siteName]['contact'].append(contact) + return ret + + +def parseHN(data): + parsedHNs = [] + for line in data.split('\n'): + if not len(line): continue + if ':' in line: line = line[:line.find(':')] + parsedHNs.append(line) + return parsedHNs + +def findDomain (h,idx) : + h=h.split('.')[::-1] + hlen=len(h) + #if idx > hlen - 1 : + # return '' + d='' + j=0 + #for i in h[idx:] : + for i in h[:idx] : + if j == 0 : d = i + else : d = d + '.' + i + j = j + 1 + return d + +def getIP(host_name): + try: + #host_name = socket.gethostname() + host_ip = socket.gethostbyname(host_name) + #print("Hostname : ",host_name) + #print("IP : ",host_ip) + return host_ip + except: + return "Unable.to.get.an.IP" + +def getIPReversed(host_name): + try: + ip=[ele for ele in reversed(getIP(host_name).split('.'))] + ipr='' + for p in ip : + if ipr : ipr=ipr+"."+p + else : ipr=ipr + p + return ipr + except: + return "Unable.to.get.a.Reverse.IP" + +def findSitenameOld(h): + mapHostSitename = [] + #print (' h ',h ) + ip='' + jlen = len(h.split('.')) + jd = h.split('.')[-2:] + jd.sort() + if 'in2p3.fr' in h : + #print ("findSitenames getIP for ",h) + ip=getIPReversed(h) + #print ( getIP(h), ip ) + #print ("findSitenames getIP for ",h, " is ",ip) + jlen = len(ip.split('.')) + jd = ip.split('.')[-2:] + jd.sort() + #print (' h ip jlen jd ',h,ip,jlen,jd) + + nmatch=0 + matched_site='' + matched_domain='' + for site in sites.keys() : + sited = sites[site]['hosts'][0].split(".")[-2:] + if 'in2p3.fr' in h : sited = getIPReversed ( sites[site]['hosts'][0] ).split(".")[-2:] + for idx in range ( len ( sites[site]['hosts'] ) ): + sited = sites[site]['hosts'][idx].split(".")[-2:] + sited.sort() + if 'in2p3.fr' in h : + sited = getIPReversed ( sites[site]['hosts'][idx] ).split(".")[-2:] + sited.sort() + #if site == 'T2_US_Vanderbilt' : + #print ( 'host ',sites[site]['hosts'][idx], ' jd ',jd, ' sited ',sited ) + if jd == sited : break + #if site == 'T2_US_Vanderbilt' : print (site, ' length ',len(sites[site]['flavors']), ' jd ',jd, ' sited ',sited) + if jd != sited : continue + if not "XROOTD" in set(sites[site]['flavors']) : continue + #socket.gethostbyname('sbgse1.in2p3.fr') + for i in range(len(sites[site]['flavors'])) : + for j in range(1, jlen): + adomain=findDomain(h,j) + bdomain=findDomain(sites[site]['hosts'][i],j) + if 'in2p3.fr' in h : + adomain=findDomain( ip , j) + bdomain=findDomain( getIPReversed ( sites[site]['hosts'][i] ), j) + + #print ( " j ", j, " adomain ",adomain, " bdomain ",bdomain) + if adomain == bdomain : + #print ( " returns j ", j, " adomain ",adomain, " bdomain ",bdomain) + #len(adomain.split('.')) + #if jlen - j > nmatch : + if len(adomain.split('.')) > nmatch : + matched_site=sites[site]['sites'][i] + nmatch = len(adomain.split('.')) # jlen - j + matched_domain=findDomain(adomain,nmatch) + #if jd == sited : print (h, site) + #if mapHostSitename : mapHostSitename[h] = [] + #print ( "appending site ",site) + mapHostSitename.append ( site ) + #mapHostSitename.append ( "m="+matched_site ) + #mapHostSitename.append ( matched_site ) + #if len( mapHostSitename ) > 1 and len ( matched_domain.split(".") ) < 3 : + # print ( h, matched_site, mapHostSitename) + return (matched_site, matched_domain) + +def findSitename(h): + global sites + gredirs = getGlobalRedirectors() + for idx in range ( len (gredirs['hosts']) ) : + if h in gredirs['hosts'][idx] : + return ( gredirs['sites'][idx], 'TOBEFIXED') + + mapHostSitename = [] + #print (' h ',h ) + ip='' + jlen = len(h.split('.')) + jd = h.split('.')[-2:] + jd.sort() + if 'in2p3.fr' in h : + #print ("findSitenames getIP for ",h) + ip=getIPReversed(h) + #print ( getIP(h), ip ) + #print ("findSitenames getIP for ",h, " is ",ip) + jlen = len(ip.split('.')) + jd = ip.split('.')[-2:] + jd.sort() + #print (' h ip jlen jd ',h,ip,jlen,jd) + + nmatch=0 + matched_site='' + matched_domain='' + for stype in [ 'xrootd','all' ] : + for site in sites.keys() : + sited = sites[site]['hosts'][0].split(".")[-2:] + if 'in2p3.fr' in h : sited = getIPReversed ( sites[site]['hosts'][0] ).split(".")[-2:] + for idx in range ( len ( sites[site]['hosts'] ) ): + sited = sites[site]['hosts'][idx].split(".")[-2:] + sited.sort() + if 'in2p3.fr' in h : + sited = getIPReversed ( sites[site]['hosts'][idx] ).split(".")[-2:] + sited.sort() + #if site == 'T2_US_Vanderbilt' : + #print ( 'host ',sites[site]['hosts'][idx], ' jd ',jd, ' sited ',sited ) + if jd == sited : break + #if site == 'T2_US_Vanderbilt' : print (site, ' length ',len(sites[site]['flavors']), ' jd ',jd, ' sited ',sited) + if jd != sited : continue + #print (" jd found ",jd ) + if stype == 'xrootd' : + if not "XROOTD" in set(sites[site]['flavors']) : continue + #socket.gethostbyname('sbgse1.in2p3.fr') + for i in range(len(sites[site]['flavors'])) : + for j in range(1, jlen): + adomain=findDomain(h,j) + bdomain=findDomain(sites[site]['hosts'][i],j) + if 'in2p3.fr' in h : + adomain=findDomain( ip , j) + bdomain=findDomain( getIPReversed ( sites[site]['hosts'][i] ), j) + + #print ( " j ", j, " adomain ",adomain, " bdomain ",bdomain) + if adomain == bdomain : + #print ( " returns j ", j, " adomain ",adomain, " bdomain ",bdomain) + #len(adomain.split('.')) + #if jlen - j > nmatch : + if len(adomain.split('.')) > nmatch : + matched_site=sites[site]['sites'][i] + nmatch = len(adomain.split('.')) # jlen - j + matched_domain=findDomain(adomain,nmatch) + #if jd == sited : print (h, site) + #if mapHostSitename : mapHostSitename[h] = [] + #print ( "appending site ",site) + #mapHostSitename.append ( site ) + #mapHostSitename.append ( "m="+matched_site ) + #mapHostSitename.append ( matched_site ) + if matched_site : break + + #if len( mapHostSitename ) > 1 and len ( matched_domain.split(".") ) < 3 : + # print ( h, matched_site, mapHostSitename) + return (matched_site, matched_domain) + +def updateHostSitename ( h, site ) : + #HostSitenames[h] = {} + HostSitenames[h] = site + with open(THEPATH+'mapHostSitenames.py','w') as data: + data.write('HostSitenames = ' + str(HostSitenames)) + +def updateSiteStorage ( site , storage ) : + #SiteStorages[ site ] = {} + SiteStorages[ site ] = storage + with open(THEPATH+'siteStoages.py','w') as data: + data.write('SiteStorages = ' + str(SiteStorages)) + + +def getRegionalRedirectors (): + global xrdmapc_output_prod + + #Lines=[] + #with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() + regionalRedirectors = {} + count = 0 + #for line in set(Lines) : + for line in xrdmapc_output_prod.splitlines() : + line = line.decode() # python3 + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #if 'Man' in entries[1] : + if entries[1] == 'Man' : + #print ( "found Man in ",entries[1] ) + if int (entries[0]) == 1 : + endpoint = entries[2] + host = endpoint.split(':')[0] + if host in HostSitenames : + site=HostSitenames[host] + else : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + regionalRedirectors[site] = {} + regionalRedirectors[site]['endpoint'] = endpoint + regionalRedirectors[site]['host'] = host + regionalRedirectors[site]['flavors'] = 'XROOTD' + regionalRedirectors[site]['federation'] = 'prod' + + #count += 1 + #print ( regionalRedirectors ) + return regionalRedirectors + +def getRegionalRedirectorsFromFile (): + Lines=[] + with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() + regionalRedirectors = {} + #count = 0 + for line in set(Lines) : + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + if entries[1] == 'Man' : + if int (entries[0]) == 1 : + endpoint = entries[2] + host = endpoint.split(':')[0] + if host in HostSitenames : + site=HostSitenames[host] + else : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + regionalRedirectors[site] = {} + regionalRedirectors[site]['endpoint'] = endpoint + regionalRedirectors[site]['host'] = host + regionalRedirectors[site]['flavors'] = 'XROOTD' + regionalRedirectors[site]['federation'] = 'prod' + + #count += 1 + #print ( regionalRedirectors ) + return regionalRedirectors + + +def getXrootdServers () : + global xrdmapc_output_prod + global sites + #Lines=[] + #with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() + #print ("DEBUG sites ",len ( sites )) + xrdServers = {} + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + count = 0 + for line in xrdmapc_output_prod.splitlines() : + #print("Line{}: {}".format(count, line.split())) + line = line.decode() + entries=line.split() + #print ( entries ) + #if entries[0] == 'Srv' : + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + if '[' in endpoint : continue + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if host in HostSitenames : + site=HostSitenames[host] + else : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + #print ( host, site ) + #print ( xrdServers[site]['endpoints'] ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # site = 'T2_IN_TIFR' + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + # xrdServers[site]['hosts'].append ( 'se01.indiacms.res.in' ) + # xrdServers[site]['federation'].append ( 'prod' ) + #print ( endpoint, host, site ) + #if 'unl.edu' in endpoint : print ( endpoint, host , site, domain) + # addhoc + #if not site : + # #if 'vanderbilt.edu' in host : site = 'T2_US_Vanderbilt' + + #xrdServers[site] = {} + #print ( " site ",site) + #if site == 'T2_US_Nebraska' : print ( 'T2_US_Nebraska ', endpoint ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + #else : + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'prod' ) + #print("Line{}: {}".format(count, line.split()), ' idx = ',idx, endpoint,site,xrdServers[site]['federation']) + + #count += 1 + return xrdServers + +def getXrootdServersFromFile () : + global sites + Lines=[] + with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() + print ("DEBUG sites ",len ( sites )) + xrdServers = {} + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + count = 0 + for line in Lines : + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #print ( entries ) + #if entries[0] == 'Srv' : + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + if '[' in endpoint : continue + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if host in HostSitenames : + site=HostSitenames[host] + else : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + #print ( host, site ) + #print ( xrdServers[site]['endpoints'] ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # site = 'T2_IN_TIFR' + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + # xrdServers[site]['hosts'].append ( 'se01.indiacms.res.in' ) + # xrdServers[site]['federation'].append ( 'prod' ) + #print ( endpoint, host, site ) + #if 'unl.edu' in endpoint : print ( endpoint, host , site, domain) + # addhoc + #if not site : + # #if 'vanderbilt.edu' in host : site = 'T2_US_Vanderbilt' + + #xrdServers[site] = {} + #print ( " site ",site) + #if site == 'T2_US_Nebraska' : print ( 'T2_US_Nebraska ', endpoint ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + #else : + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'prod' ) + #print("Line{}: {}".format(count, line.split()), ' idx = ',idx, endpoint,site,xrdServers[site]['federation']) + + #count += 1 + return xrdServers + +def getTransitionalXrootds () : + global sites + global xrdmapc_output_tran + #Lines=[] + #with open(THEPATH+'out/xrdmapc_trans_3.txt') as f: Lines = f.readlines() + xrdServers = {} + #print ("DEBUG sites ",sites.keys()) + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + #count = 0 + for line in xrdmapc_output_tran.splitlines() : + line = line.decode() + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #print ( entries + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if '[' in endpoint: continue + + if host in HostSitenames : + #print (' if host = ',host, ' HostSitenames ',HostSitenames ) + site=HostSitenames[host] + if not site : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + else : + #print (' else host = ',host ) + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + #print (' else host = ',host, ' site ',site ) + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'trans' ) + #print (' endpoint ',endpoint ) + + #count += 1 + return xrdServers + +def getXrootdsFromTransFile () : + global sites + Lines=[] + with open(THEPATH+'out/xrdmapc_trans_3.txt') as f: Lines = f.readlines() + xrdServers = {} + #print ("DEBUG sites ",sites.keys()) + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + #count = 0 + for line in Lines : + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #print ( entries + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if '[' in endpoint: continue + + if host in HostSitenames : + #print (' if host = ',host, ' HostSitenames ',HostSitenames ) + site=HostSitenames[host] + if not site : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + else : + #print (' else host = ',host ) + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + #print (' else host = ',host, ' site ',site ) + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'trans' ) + #print (' endpoint ',endpoint ) + + #count += 1 + return xrdServers + +def queryXrdmapc (redirector) : + global sites + global xrdmapc_output_prod + global xrdmapc_output_tran + if 'trans' in redirector : + (err_info,xrdmapc_output_tran,dump_info) = xrd_info(redirector,'') + else : + (err_info,xrdmapc_output_prod,dump_info) = xrd_info(redirector,'') + +def updateXrdInfo (thesite) : + global sites + xrdInfo = sites[thesite] + xrd_idxes = findXROOTDIdxes ( xrdInfo ) + #print ( thesite, sites[thesite] ) + for idx in range(len(xrd_idxes)) : + endpoint = sites[thesite]['endpoints'][xrd_idxes[idx]] + (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + #xrdInfo['xrootd_version'][xrd_idxes[idx]] = xrootd_version + sites[thesite]['xrootd_version'][xrd_idxes[idx]] = xrootd_version + (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + #xrdInfo['xrootd_role'][xrd_idxes[idx]] = xrootd_role + sites[thesite]['xrootd_role'][xrd_idxes[idx]] = xrootd_role + if not 'timeout' in xrootd_role and not 'FATAL' in xrootd_role : + if not 'timeout' in xrootd_version and not 'FATAL' in xrootd_version : + updatexrdVersions ( endpoint, xrootd_role, xrootd_version ) + #if not xrootd_version in XrdVersions[endpoint]['xrootd_version'] or not xrootd_role in XrdVersions[endpoint]['xrootd_role'] : + # updatexrdVersions ( endpoint, xrootd_role, xrootd_version ) + from xrdVersions import XrdVersions + if endpoint in XrdVersions : + if 'timeout' in xrootd_role or 'FATAL' in xrootd_role : + #print ( "( endpont )",endpoint ) + xrootd_role = XrdVersions[endpoint]['Role'] + sites[thesite]['xrootd_role'][xrd_idxes[idx]] = '['+xrootd_role+']' + if 'timeout' in xrootd_version or 'FATAL' in xrootd_version : sites[thesite]['xrootd_version'][xrd_idxes[idx]] = '['+XrdVersions[endpoint]['Version']+']' + +#def updateFederation (thesite) : +# xrdInfo = sites[thesite] +# xrd_idxes = findXROOTDIdxes ( xrdInfo ) +# #print ( thesite, sites[thesite] ) +# for idx in range(len(xrd_idxes)) : +# endpoint = sites[thesite]['endpoints'][xrd_idxes[idx]] +# (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") +# #xrdInfo['xrootd_version'][xrd_idxes[idx]] = xrootd_version +# sites[thesite]['federation'][xrd_idxes[idx]] = xrootd_version +# (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") +# #xrdInfo['xrootd_role'][xrd_idxes[idx]] = xrootd_role +# sites[thesite]['xrootd_role'][xrd_idxes[idx]] = xrootd_role + +def updateContactInfo () : + global sites + for site in sites : + xrdInfo = sites[site] + xrd_idxes = findXROOTDIdxes ( xrdInfo ) + contact = '' + for contact in sites[site]['contact'] : + if '@' in contact : break + for idx in range(len(xrd_idxes)) : + #sites[site]['contact'][xrd_idxes[idx]] = contact + sites[site]['contact'][xrd_idxes[idx]] = contact.replace('@cern.ch','') + +def findXROOTDIdxes (thesite) : + idxes=[] + i=0 + for flavor in thesite['flavors'] : + if flavor == 'XROOTD' : idxes.append(i) + i += 1 + if len(idxes) == 0 : + i=0 + for flavor in thesite['flavors'] : + if flavor == 'WEBDAV' : idxes.append(i) + i += 1 + + return idxes +def runqueryXrdmapc () : + signal.signal(signal.SIGALRM, alarm_handler) + try: + threads = list() + for redirector in topRedirectors: + if 1 == 0 : + queryXrdmapc (redirector+':1094') + else : + t = threading.Thread(target=queryXrdmapc, args=(redirector+':1094',)) + threads.append(t) + t.start() + print ("Start joining threads ") + for xrd, thread in enumerate(threads): + #print ("Before joining thread ", site) + thread.join() + print ("All threads done ") + except Alarm: + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + #clear_lock() + #sys.exit(2) + signal.alarm(0) + +def runThreadedupdateXrdInfo () : + + signal.signal(signal.SIGALRM, alarm_handler) + try: + threads = list() + for site in sites: + if 1 == 0 : + updateXrdInfo (site) + else : + t = threading.Thread(target=updateXrdInfo, args=(site,)) + threads.append(t) + t.start() + print ("Start joining threads ") + for site, thread in enumerate(threads): + #print ("Before joining thread ", site) + thread.join() + print ("All threads done ") + except Alarm: + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + #clear_lock() + #sys.exit(2) + signal.alarm(0) + + +#with open(THEPATH+'exceptions.json') as f: exc = f.read() +#exc = json.loads(exc) + +#def exception(name): +# ret = None +# for i in exc.keys(): +# if i == name : return exc[i]['VOname'] +# return ret + + +#def siteName2CMSSiteName(dom): +# ret = None +# for cmsSite in sites.keys(): +# ret = exception(dom) +# if ret : +# ret = str(ret) +# return ret +# if sites[cmsSite]['hosts'][0].find(dom) != -1: +# return cmsSite +# return ret + +def jsonSorting(item): + if isinstance(item, dict): + return sorted((key, jsonSorting(values)) for key, values in item.items()) + if isinstance(item, list): + return sorted(jsonSorting(x) for x in item) + else: + return item + +def getAllSiteRedirectors () : + global sites + try : + print ( 'DEBUG Doing getSites' ) + sites = getSites() + + print ( 'DEBUG Doing gRedirs ') + gredirs = getGlobalRedirectors() + + print ( 'DEBUG Doing rRedirs ') + #regionalRedirectors = getRegionalRedirectorsFromFile() + regionalRedirectors = getRegionalRedirectors () + #print ( 'DEBUG regionalRedirectors ', regionalRedirectors ) + print ( 'DEBUG Doing xrdServers ') + #xrdProdServers = getXrootdServersFromFile() + xrdProdServers = getXrootdServers() + #print ( 'DEBUG xrdProdServers ', xrdProdServers ) + print ( 'DEBUG Doing xrd from trans redirector ') + #xrdTranServers = getXrootdsFromTransFile() + xrdTranServers = getTransitionalXrootds() + #print ( 'DEBUG xrdTranServers ', xrdTranServers) + for site in sites.keys(): + #print ( 'DEBUG Doing site g ',site ) + if 'T0_CH_CERN' in site : + for sitekey in sites[site].keys() : + for siteval in gredirs[sitekey] : + sites[site][sitekey].append(siteval) + #print ( 'Doing site r ',site ) + #if 'cmsxrootd-site3.fnal.gov:1094' in sites[site]['endpoints'] : + #print ( ' before regional ', site, ' has it in ', sites[site]['endpoints'] ) + if site in regionalRedirectors : #.has_key(site): + for sitekey in sites[site].keys() : + + #print (" site RR ",site, " sitekey ",sitekey, sites[site][sitekey]) + + if 'sites' in sitekey : sites[site][sitekey].append( site ) + elif 'hosts' in sitekey : sites[site][sitekey].append( regionalRedirectors[site]['host'] ) + elif 'flavors' in sitekey : sites[site][sitekey].append( 'XROOTD' ) + elif 'endpoints' in sitekey : sites[site][sitekey].append( regionalRedirectors[site]['endpoint'] ) + elif 'name' in sitekey : sites[site][sitekey].append( 'RR_service_from_'+site ) + elif 'xrootd_version' in sitekey : + #(err_info,xrootd_version,dump_info) = xrd_info(regionalRedirectors[site]['endpoint'],"version") + sites[site][sitekey].append( '' ) + elif 'xrootd_role' in sitekey : + #(err_info,xrootd_role,dump_info) = xrd_info(regionalRedirectors[site]['endpoint'],"role") + sites[site][sitekey].append( '' ) + elif 'xrootd_servers' in sitekey : sites[site][sitekey].append( 'RR' ) + elif 'xrootd_storage' in sitekey : + if SiteStorages[ site ] : xrootd_storage = SiteStorages[ site ] + else : + xrootd_storage = getStorageFromStorageJson ( site ) + updateSiteStorage ( site , xrootd_storage ) + + sites[site][sitekey].append( xrootd_storage ) + elif 'federation' in sitekey : sites[site][sitekey].append( 'prod' ) + else : + sites[site][sitekey].append( 'RR' ) + #if 'cmsxrootd-site3.fnal.gov:1094' in sites[site]['endpoints'] : + # print ( ' after regional ', site, ' has it in ', sites[site]['endpoints'] ) + ic = 0 + for xrdServers in [ xrdProdServers, xrdTranServers ] : + #ic = ic + 1 #print ( 'Doing xrdServers ',xrdServers ) #site x ',site, ' endpoints count ', xrdServers[site]['endpoints'] ) + if site in xrdServers : #.has_key(site): + #print (' xrdServers has key ic site ',ic,site, xrdServers[site]) + for idx in range ( len ( xrdServers[site]['endpoints'] ) ) : + if xrdServers[site]['endpoints'][idx] in sites[site]['endpoints'] : + #xrdInfo = sites[site] + #if xrdServers[site]['endpoints'][idx] == 'cmsxrootd-site1.fnal.gov:1094' : print ( site, "will update sites[site]['endpoints'] for ",xrdServers[site]['endpoints'][idx], xrdInfo ) #updateFederation (site) # + #xrd_idxes = findXROOTDIdxes ( xrdInfo ) + for xrd_idx in range(len( sites[site]['endpoints'] )) : + #if xrdServers[site]['endpoints'][idx] == 'cmsxrootd-site1.fnal.gov:1094' : #updateFederation (site) # + # print ( site, "will update sites[site]['endpoints'] for ",xrdServers[site]['endpoints'][idx], ' vs ',sites[site]['endpoints'][xrd_idx]) + if xrdServers[site]['endpoints'][idx] in sites[site]['endpoints'][xrd_idx] : + #if xrdServers[site]['endpoints'][idx] == 'cmsxrootd-site1.fnal.gov:1094' : #updateFederation (site) # + # print ( site, "will update sites[site]['endpoints'] for ",xrdServers[site]['endpoints'][idx]) + #sites[site]['endpoints'][xrd_idx] = xrdServers[site]['endpoints'][idx] + #sites[site]['hosts'][xrd_idx] = xrdServers[site]['hosts'][idx] + #sites[site]['flavors'][xrd_idx] = xrdServers[site]['flavors'][idx] + sites[site]['federation'][xrd_idx] = xrdServers[site]['federation'][idx] + #print ( 'skipping site idx ',site,idx,xrdServers[site]['endpoints'][idx],sites[site]['endpoints']) + continue + #for sitekey in sites[site].keys() : + #for sitekey in sites[site].keys() : + #print ( 'Doing site x site ',site, ' n endpoints ',len ( xrdServers[site]['endpoints'] ), ' endpoint ',xrdServers[site]['endpoints'][idx]) + #for idx in range ( len ( xrdServers[site]['endpoints'] ) ) : + #if not 'sites' in sitekey : continue + #print ( 'Doing site x ',site ) + #print ( 'Doing site x site ',site, ' n endpoints ',len ( xrdServers[site]['endpoints'] ), ' idx ', idx, ' endpoint ',xrdServers[site]['endpoints'][idx]) + + sites[site]['sites'].append( site ) + sites[site]['hosts'].append( xrdServers[site]['hosts'][idx] ) + sites[site]['flavors'].append( 'XROOTD' ) + sites[site]['endpoints'].append( xrdServers[site]['endpoints'][idx] ) + sites[site]['name'].append( 'Xrootd_Servers_from_'+site ) + sites[site]['xrootd_version'].append( 'unknown' ) + sites[site]['xrootd_role'].append( 'server' ) + sites[site]['xrootd_servers'].append( xrdServers[site]['hosts'][idx] ) + xrootd_storage = 'uninitialized' + if SiteStorages[ site ] : xrootd_storage = SiteStorages[ site ] + else : + xrootd_storage = getStorageFromStorageJson ( site ) + updateSiteStorage ( site , xrootd_storage ) + + sites[site]['xrootd_storage'].append( xrootd_storage ) + sites[site]['contact'].append( site ) + sites[site]['federation'].append( xrdServers[site]['federation'][idx] ) + + #print ( 'Doing site x ',site, sites[site]) + updateContactInfo() + return sites + except Exception as e : + err={} + err["error"] = str(e) + #print (json.dumps(err)) + print ( err ) + sys.exit(1) + +#for site in sites : +# idxes = findXROOTDIdxes (sites[site]) +# #print ( ' site ', site, ' fed ',sites [ site ]['federation'], ' endpoint ',sites [ site ]['endpoints'] ) +# for idx in idxes : +# if sites [ site ]['flavors'][idx] == 'XROOTD' : +# print ( ' site ', site, 'idx ',idx, ' fed ',sites [ site ]['federation'][idx], ' endpoint ',sites [ site ]['endpoints'][idx] ) +## if 'T2_US_Florida' in site : +## for key in sites [site] : +## print ( len (sites[site][key]), key, sites[site][key] ) + +#sys.exit(0) + +# Checks if a hostname is an alias or not +def getDNSARecords ( h ): + import dns.resolver + #import socket + if h in DNSARecords : #.has_key (h) : + return ( DNSARecords[h]['Alias'], DNSARecords[h]['A'] ) + + isAlias = False + aRecords = [] + try : + result = dns.resolver.query(h, 'A') + for ipval in result: + ip = ipval.to_text() + (host, alias, ipl) = socket.gethostbyaddr(ip) + aRecords.append (host) + if len(aRecords) > 1 : + isAlias = True + #print ("Updating DNS Records for h = ", h) + updateDNSARecords ( h, isAlias, aRecords ) + return (isAlias, aRecords) + except Exception as e : + print ("Exception h ",h, " ",e) + return (isAlias, aRecords) + +def updateDNSARecords ( h, isAlias, aRecords ) : + #if not DNSARecords.has_key (h) : + DNSARecords[ h ] = {'Alias' : isAlias, 'A' : aRecords } + with open(THEPATH+'DNSARecords.py','w') as data: + #print ( 'DNSARecords = ' + str(DNSARecords) ) + data.write('DNSARecords = ' + str(DNSARecords)) + +def updatexrdVersions ( endpoint, role, version ) : + #if not DNSARecords.has_key (h) : + XrdVersions[ endpoint ] = {'Role' : role, 'Version' : version } + with open(THEPATH+'xrdVersions.py','w') as data: + data.write('XrdVersions = ' + str(XrdVersions)) + +class MyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + elif isinstance(obj, bytes): + return str(obj, encoding='utf-8'); + return json.JSONEncoder.default(self, obj) + +if __name__ == "__main__": + # run xrdmapc against the global redirector and the transitional redirector + runqueryXrdmapc () + + # get all redirectors from the hardwired way or the vofeed or the xrdmapc output + sites = getAllSiteRedirectors () + + # fill xrootd version and xrootd role in a threaded way + runThreadedupdateXrdInfo () + + # + #if 1 == 0 : + for site in sites : + idxes = findXROOTDIdxes (sites[site]) + for idx in idxes : + xrdv = sites [ site ]['xrootd_version'][idx] + endpoint = sites [ site ]['endpoints'][idx] + xrdr = sites [ site ]['xrootd_role'][idx] + if 'timeout' in xrdv : + #(err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + #(err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + #sites [ site ]['xrootd_version'][idx] = xrootd_version + print ( site, endpoint,xrdv) # " version new ",xrootd_version) + if 'timeout' in xrdr : + #(err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + #(err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + #sites [ site ]['xrootd_role'][idx] = xrootd_role + print ( site, endpoint,xrdr) # " role new ",xrootd_role) + # + + + federations={} + for site in sites : + idxes = findXROOTDIdxes (sites[site]) + for idx in idxes : + # Update if the host is an alias and the federation is nowhere + if ( 'T0_' in site or 'T1_' in site or 'T2_' in site ) and sites [ site ]['federation'][idx] == 'nowhere' : + #print ( 'Doing getDNSRecords ', sites [ site ]['hosts'][idx] ) + (isAlias, A) = getDNSARecords ( sites [ site ]['hosts'][idx] ) + if isAlias : + aFound = True + shost = '' + for ar in A : + #print ( ' ar ',ar, ' hosts ', sites [ site ]['hosts'] ) + if not ar in sites [ site ]['hosts'] : aFound = False + shost = ar + + if len (A) > 1 : + if shost in sites [ site ]['hosts'] : + sites [ site ]['federation'][idx] = sites [ site ] ['federation'][sites [ site ]['hosts'].index ( shost )] + else : + print ( ' len(A) ', sites [site]['hosts'] ) + + #print ( ' site ', site, 'idx ',idx, ' fed ',sites [ site ]['federation'][idx], ' endpoint ',sites [ site ]['endpoints'][idx] , isAlias , aFound ) + #print ( ' site ', site, 'idx ',idx, ' fed ',sites [ site ]['federation'][idx], ' endpoint ',sites [ site ]['endpoints'][idx] ) + #output[sites[site]['federation'][idx]].append(sites[site]) + federations[sites[site]['federation'][idx]] = {} + for federation in federations : + for site in sites : + idxes = findXROOTDIdxes (sites[site]) + for idx in idxes : + if sites[site]['federation'][idx] in federation : + thesite = {} + thesite['sites'] = sites[site]['sites'][idx] + thesite['hosts'] = sites[site]['hosts'][idx] + thesite['flavors'] = sites[site]['flavors'][idx] + thesite['endpoints'] = sites[site]['endpoints'][idx] + thesite['name'] = sites[site]['name'][idx] + thesite['contact'] = sites[site]['contact'][idx] + thesite['xrootd_version'] = sites[site]['xrootd_version'][idx] + thesite['xrootd_role'] = sites[site]['xrootd_role'][idx] + thesite['xrootd_servers'] = sites[site]['xrootd_servers'][idx] + thesite['xrootd_storage'] = sites[site]['xrootd_storage'][idx] + thesite['federation'] = sites[site]['federation'][idx] + output[federation].append(thesite) + #for federation in federations : + # for thesite in output[federation] : + # print ( federation, thesite ) + with open(THEPATH+'out/federations.json', 'w') as f: + #f.write(json.dumps(output, indent = 1)) + f.write(json.dumps(output,cls=MyEncoder,indent=1)) + print ("Check ",THEPATH+'out/federations.json') +exit() diff --git a/AAAOps/FedProbeSendAAAMetrics/listSrv.sh b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh new file mode 100755 index 0000000..df7416b --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for h in $( grep Srv out/xrdmapc_all_0.txt | awk '{print $2}' | cut -d: -f1 | grep -v \\[ | sort -u ) ; do + echo $h $(ping -c 1 $h | grep ^PING | cut -d\( -f2 | cut -d\) -f1) + +done diff --git a/AAAOps/FedProbeSendAAAMetrics/mapHostSitename.sh b/AAAOps/FedProbeSendAAAMetrics/mapHostSitename.sh new file mode 100755 index 0000000..ecdca97 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/mapHostSitename.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for h in $( grep Srv out/xrdmapc_all_0.txt | awk '{print $2}' | cut -d: -f1 | grep -v \\[ | sort -u ) ; do + python findSitename.py $h + break +done diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh new file mode 100755 index 0000000..fa21f2f --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -0,0 +1,54 @@ +#!/bin/bash +THEPATH=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +FED_json=$THEPATH/out/federations.json +THELOG=$THEPATH/logs/probe_create_send_aaa_metrics.log +KIBANA_PAGE=https://monit-kibana.cern.ch/kibana/goto/5d1128ff8482ac3b00e4be3d5a06e954 + +#bockjoo original export X509_USER_PROXY=/root/.globus/slsprobe.proxy +export X509_USER_PROXY=$HOME/.cmsuser.proxy #bockoo comment this out when in production +notifytowhom=bockjoo@gmail.com +export PYTHONPATH=$PYTHONPATH:$THEPATH/CMSMonitoring/src/python/ + +[ -d $THEPATH/out ] || mkdir -p $THEPATH/out + +python3 $THEPATH/create_fedmaps.py > $THEPATH/create_fedmaps.log 2>&1 + +if [ ! -r $FED_json ]; then + echo "We have a problem creating JSON file.\n" + printf "$(/bin/hostname -s) $(basename $0) We have a problem creating JSON file.\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + exit 1 +fi + +cat $FED_json | python -m json.tool 2>/dev/null 1>/dev/null +if [ $? -ne 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $FED_json.\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $FED_json + exit 1 +fi +[ -d $(dirname $THELOG) ] || mkdir -p $(dirname $THELOG) + +rm -f $THEPATH/fed.json +python3 $THEPATH/aaa_federation.py --amq $THEPATH/credentials.json > $THEPATH/aaa_federation.log 2>&1 +status=$? +if [ $(date +%M ) -lt 15 ] ; then + if [ $status -eq 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + else + printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + fi +fi +nprod_exp=55 +if [ -f $THEPATH/check_subscribed_sites.sh ] ; then + nprod=$($THEPATH/check_subscribed_sites.sh | tail -1 | awk '{print $2}') + if [ $nprod_exp -gt $nprod ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + #exit 1 + fi + echo "Sites subscribed to the Production Federation: " $nprod +fi + +[ -f ${THELOG}_$(date +%Y%m%d) ] || touch ${THELOG}_$(date +%Y%m%d) +if [ -f $THELOG ] ; then + cat $THELOG >> ${THELOG}_$(date +%Y%m%d) +fi + +exit $status From 6917863bd135a6979ce7f7bc642b5f2194bc3dc9 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 25 Aug 2021 14:12:30 -0400 Subject: [PATCH 065/192] Obscure email a bit --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index fa21f2f..bafc6b7 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -5,9 +5,9 @@ THELOG=$THEPATH/logs/probe_create_send_aaa_metrics.log KIBANA_PAGE=https://monit-kibana.cern.ch/kibana/goto/5d1128ff8482ac3b00e4be3d5a06e954 #bockjoo original export X509_USER_PROXY=/root/.globus/slsprobe.proxy -export X509_USER_PROXY=$HOME/.cmsuser.proxy #bockoo comment this out when in production -notifytowhom=bockjoo@gmail.com +notifytowhom=bockjoo__AT__gmail__dot__com export PYTHONPATH=$PYTHONPATH:$THEPATH/CMSMonitoring/src/python/ +notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#") [ -d $THEPATH/out ] || mkdir -p $THEPATH/out From 237d5b695faced01266973a66d859fe65d61af26 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 25 Aug 2021 14:14:08 -0400 Subject: [PATCH 066/192] Obscure the email a bit --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index bafc6b7..6fa3cfa 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -7,7 +7,7 @@ KIBANA_PAGE=https://monit-kibana.cern.ch/kibana/goto/5d1128ff8482ac3b00e4be3d5a0 #bockjoo original export X509_USER_PROXY=/root/.globus/slsprobe.proxy notifytowhom=bockjoo__AT__gmail__dot__com export PYTHONPATH=$PYTHONPATH:$THEPATH/CMSMonitoring/src/python/ -notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#") +notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') [ -d $THEPATH/out ] || mkdir -p $THEPATH/out From 725045cd0e99d546bf58057aec983d11dc3f6eb9 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 25 Aug 2021 14:22:10 -0400 Subject: [PATCH 067/192] Change the proxy default --- .../probe_create_send_aaa_metrics.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 6fa3cfa..009c67e 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -4,7 +4,14 @@ FED_json=$THEPATH/out/federations.json THELOG=$THEPATH/logs/probe_create_send_aaa_metrics.log KIBANA_PAGE=https://monit-kibana.cern.ch/kibana/goto/5d1128ff8482ac3b00e4be3d5a06e954 -#bockjoo original export X509_USER_PROXY=/root/.globus/slsprobe.proxy +export X509_USER_PROXY=$HOME/.globus/slsprobe.proxy +if [ -f $X509_USER_PROXY ] ; then + if [ $(voms-proxy-info -timeleft 2>/dev/null) -lt 3600 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\n$(voms-proxy-info -all | sed 's#%#%%#g')" | mail -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 1" $notifytowhom + fi +else + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\nIt does not exist" | mail -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 2" $notifytowhom +fi notifytowhom=bockjoo__AT__gmail__dot__com export PYTHONPATH=$PYTHONPATH:$THEPATH/CMSMonitoring/src/python/ notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') From a9d01ce0e16fec26ac91dbe648d1a293496be28f Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 30 Aug 2021 10:07:33 -0400 Subject: [PATCH 068/192] Write xrdmapc output to a file --- AAAOps/FedProbeSendAAAMetrics/README | 6 +++- .../FedProbeSendAAAMetrics/create_fedmaps.py | 27 +++++++++++++++--- AAAOps/FedProbeSendAAAMetrics/listSrv.sh | 6 +++- .../probe_create_send_aaa_metrics.sh | 28 +++++++++++++------ 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/README b/AAAOps/FedProbeSendAAAMetrics/README index d8bb766..f7432d2 100644 --- a/AAAOps/FedProbeSendAAAMetrics/README +++ b/AAAOps/FedProbeSendAAAMetrics/README @@ -17,9 +17,13 @@ listSrv.sh check_xrdmapc_from_vofeed.sh check_vofeed_from_xrdmapc.sh check_upstream_redirector_examples.sh -check_production_sites.sh +check_subscribed_sites.sh [4] Installing +# First check the needed python modules +python3 -c "import numpy" +python3 -c "import stomp" +python3 -c "import dns.resolver" # 1 cd /opt git clone git@github.com:CMSCompOps/TransferTeam.git diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index 32f277c..997c374 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -50,10 +50,11 @@ output = {"prod" : [], "trans" : [], "nowhere" : []} sites = {} xrdmapc_output_prod = '' +xrdmapc_output_prod_2 = '' xrdmapc_output_tran = '' gRedirectors=['cms-xrd-global01.cern.ch', 'cms-xrd-global02.cern.ch'] tRedirectors=['cms-xrd-transit.cern.ch'] -topRedirectors = [gRedirectors[0],]+tRedirectors +topRedirectors = [gRedirectors[0],gRedirectors[1],]+tRedirectors if not path.exists(THEPATH+'mapHostSitenames.py') : with open(THEPATH+'mapHostSitenames.py','w') as data : data.write('HostSitenames = {}') @@ -686,6 +687,7 @@ def updateSiteStorage ( site , storage ) : def getRegionalRedirectors (): global xrdmapc_output_prod + global xrdmapc_output_prod_2 #Lines=[] #with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() @@ -747,9 +749,9 @@ def getRegionalRedirectorsFromFile (): def getXrootdServers () : global xrdmapc_output_prod + global xrdmapc_output_prod_2 global sites #Lines=[] - #with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() #print ("DEBUG sites ",len ( sites )) xrdServers = {} for site in sites : @@ -782,7 +784,7 @@ def getXrootdServers () : (site,domain) = findSitename(host) updateHostSitename ( host, site ) #print ( host, site ) - #print ( xrdServers[site]['endpoints'] ) + #print ( "DEBUG xrdServers[site]['endpoints'] ", xrdServers[site]['endpoints'] ) #if '[::ffff:144.16.111.9]:11001' in endpoint : # site = 'T2_IN_TIFR' # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') @@ -978,11 +980,27 @@ def getXrootdsFromTransFile () : def queryXrdmapc (redirector) : global sites global xrdmapc_output_prod + global xrdmapc_output_prod_2 global xrdmapc_output_tran if 'trans' in redirector : (err_info,xrdmapc_output_tran,dump_info) = xrd_info(redirector,'') + with open(THEPATH+'out/xrdmapc_tran.txt','w' ) as f: + for line in xrdmapc_output_tran.splitlines() : + f.write(str(line.decode())) + f.write('\n') + elif 'global02' in redirector : + (err_info,xrdmapc_output_prod_2,dump_info) = xrd_info(redirector,'') + with open(THEPATH+'out/xrdmapc_prod_2.txt','w' ) as f: + for line in xrdmapc_output_prod_2.splitlines() : + f.write(str(line.decode())) + f.write('\n') else : (err_info,xrdmapc_output_prod,dump_info) = xrd_info(redirector,'') + with open(THEPATH+'out/xrdmapc_prod_1.txt','w' ) as f: + for line in xrdmapc_output_prod.splitlines() : + f.write(str(line.decode())) + f.write('\n') + def updateXrdInfo (thesite) : global sites @@ -1257,7 +1275,8 @@ def getDNSARecords ( h ): import dns.resolver #import socket if h in DNSARecords : #.has_key (h) : - return ( DNSARecords[h]['Alias'], DNSARecords[h]['A'] ) + if DNSARecords[h]['Alias'] == True : + return ( DNSARecords[h]['Alias'], DNSARecords[h]['A'] ) isAlias = False aRecords = [] diff --git a/AAAOps/FedProbeSendAAAMetrics/listSrv.sh b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh index df7416b..923006a 100755 --- a/AAAOps/FedProbeSendAAAMetrics/listSrv.sh +++ b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh @@ -1,5 +1,9 @@ #!/bin/bash +ipv= ; ipos=2 +[ $# -gt 0 ] && { ipv=6 ; ipos=3 ; } ; + for h in $( grep Srv out/xrdmapc_all_0.txt | awk '{print $2}' | cut -d: -f1 | grep -v \\[ | sort -u ) ; do - echo $h $(ping -c 1 $h | grep ^PING | cut -d\( -f2 | cut -d\) -f1) + echo $h $(ping${ipv} -c 1 $h | grep ^PING | cut -d\( -f${ipos} | cut -d\) -f1) + #sleep 1 done diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 009c67e..431cd59 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -36,23 +36,35 @@ fi rm -f $THEPATH/fed.json python3 $THEPATH/aaa_federation.py --amq $THEPATH/credentials.json > $THEPATH/aaa_federation.log 2>&1 status=$? -if [ $(date +%M ) -lt 15 ] ; then - if [ $status -eq 0 ] ; then - printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log - else - printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log - fi +# Check any error from xrdmapc +xrdmapc_error=$(grep \\[ $THEPATH/out/xrdmapc_prod_1.txt | grep -v "invalid addr" |sort -u) +xrdmapc_port0_error=$(grep ":0$\|:0 " $THEPATH/out/xrdmapc_prod_1.txt | awk '{if ($2 == "Man") print $3 ; else print $2}' | sed 's#:0$##g' | while read h ; do grep -v $h:0 $THEPATH/out/xrdmapc_prod_1.txt | grep -q $h $THEPATH/out/xrdmapc_prod_1.txt ; [ $? -eq 0 ] || echo ${h}:0 ; done) +if [ $status -eq 0 ] ; then + if [ $(date +%M ) -lt 15 ] ; then + printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + fi +else + printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log fi + nprod_exp=55 if [ -f $THEPATH/check_subscribed_sites.sh ] ; then - nprod=$($THEPATH/check_subscribed_sites.sh | tail -1 | awk '{print $2}') + subscribed_sites=$($THEPATH/check_subscribed_sites.sh) + nprod=$(printf "$subscribed_sites\n" | tail -1 | awk '{print $2}') + printf "$subscribed_sites\n" > $THEPATH/subscribed_sites_$nprod.txt if [ $nprod_exp -gt $nprod ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + thediff= + if [ -f $THEPATH/subscribed_sites_$nprod_exp.txt ] ; then + thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2) + fi + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$thediff\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #exit 1 fi echo "Sites subscribed to the Production Federation: " $nprod fi +printf "$(/bin/hostname -s) $(basename $0) \nxrdmapc errors:\n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" + [ -f ${THELOG}_$(date +%Y%m%d) ] || touch ${THELOG}_$(date +%Y%m%d) if [ -f $THELOG ] ; then cat $THELOG >> ${THELOG}_$(date +%Y%m%d) From b3ed20b9f04c8534c63d129d5e0964e9035dad5c Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 30 Aug 2021 11:27:07 -0400 Subject: [PATCH 069/192] stomp.py import issue fix --- AAAOps/FedProbeSendAAAMetrics/aaa_federation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py index 90079d0..068429c 100755 --- a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py +++ b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py @@ -13,10 +13,15 @@ import uuid from itertools import islice import hashlib -import stomp +#import stomp from uuid import uuid4 from urllib.request import urlopen import time +import re +__requires__ = 'stomp.py==4.1.21' + +from pkg_resources import load_entry_point +import stomp #print("stomp version", stomp.__version__) From b99aff7caa861dc6fecfef32e84d3c1ed5ac48ed Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 30 Aug 2021 11:55:25 -0400 Subject: [PATCH 070/192] fix stomp import --- AAAOps/FedProbeSendAAAMetrics/aaa_federation.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py index 068429c..8c2bc32 100755 --- a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py +++ b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py @@ -1,7 +1,13 @@ #!/usr/bin/env python +import re +import sys + +__requires__ = 'stomp.py==4.1.21' + +from pkg_resources import load_entry_point +import stomp import os -import sys import subprocess import urllib import json @@ -13,17 +19,11 @@ import uuid from itertools import islice import hashlib -#import stomp from uuid import uuid4 from urllib.request import urlopen import time -import re -__requires__ = 'stomp.py==4.1.21' - -from pkg_resources import load_entry_point -import stomp -#print("stomp version", stomp.__version__) +print("stomp version", stomp.__version__) FedProbeSendAAAMetrics = '/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' FedProbeSendAAAMetrics = '/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' From 0c5b1258418c650ba90bb205c79a3e09e666f6c4 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 3 Sep 2021 16:59:22 -0400 Subject: [PATCH 071/192] fix auth fails output --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 4 +++- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index 997c374..b66266a 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -126,8 +126,10 @@ def xrd_info(redirector,what): if out.replace(b'\n',b'') : break time.sleep(1) count += 1 - if not config_out : config_out = 'timeout' #print ( 'DEBUG count = ',count,' config_out ',config_out, ' redirector ',redirector) + if not 'xrdmapc' in command : + if not config_out : config_out = 'timeout' + if 'Auth failed' in config_out : config_out = 'Auth failed' return (errtext,config_out,out) def run_xrd_commands(cmd,args,timelimit): diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 431cd59..c456594 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -3,6 +3,7 @@ THEPATH=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics FED_json=$THEPATH/out/federations.json THELOG=$THEPATH/logs/probe_create_send_aaa_metrics.log KIBANA_PAGE=https://monit-kibana.cern.ch/kibana/goto/5d1128ff8482ac3b00e4be3d5a06e954 +GRAFANA_PAGE=https://monit-grafana.cern.ch/goto/Grgw5Q47z export X509_USER_PROXY=$HOME/.globus/slsprobe.proxy if [ -f $X509_USER_PROXY ] ; then @@ -41,10 +42,10 @@ xrdmapc_error=$(grep \\[ $THEPATH/out/xrdmapc_prod_1.txt | grep -v "invalid addr xrdmapc_port0_error=$(grep ":0$\|:0 " $THEPATH/out/xrdmapc_prod_1.txt | awk '{if ($2 == "Man") print $3 ; else print $2}' | sed 's#:0$##g' | while read h ; do grep -v $h:0 $THEPATH/out/xrdmapc_prod_1.txt | grep -q $h $THEPATH/out/xrdmapc_prod_1.txt ; [ $? -eq 0 ] || echo ${h}:0 ; done) if [ $status -eq 0 ] ; then if [ $(date +%M ) -lt 15 ] ; then - printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log fi else - printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log fi nprod_exp=55 From a37120fe84705658cc707641bf957219e7513abd Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 3 Sep 2021 20:55:53 -0400 Subject: [PATCH 072/192] change number of expected production subscription to 54 --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index c456594..c9ffa0e 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -48,7 +48,8 @@ else printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log fi -nprod_exp=55 +#nprod_exp=55 +nprod_exp=54 if [ -f $THEPATH/check_subscribed_sites.sh ] ; then subscribed_sites=$($THEPATH/check_subscribed_sites.sh) nprod=$(printf "$subscribed_sites\n" | tail -1 | awk '{print $2}') From 6704cd863fdd2f65565491c2c106fe1468ab82b2 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 3 Sep 2021 21:12:07 -0400 Subject: [PATCH 073/192] Add Grafana page for emails --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index c9ffa0e..f1663ce 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -3,7 +3,7 @@ THEPATH=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics FED_json=$THEPATH/out/federations.json THELOG=$THEPATH/logs/probe_create_send_aaa_metrics.log KIBANA_PAGE=https://monit-kibana.cern.ch/kibana/goto/5d1128ff8482ac3b00e4be3d5a06e954 -GRAFANA_PAGE=https://monit-grafana.cern.ch/goto/Grgw5Q47z +GRAFANA_PAGE="https://monit-grafana.cern.ch/d/5njhdTrWk/site-subscription?from=now-7d&orgId=11&to=now" export X509_USER_PROXY=$HOME/.globus/slsprobe.proxy if [ -f $X509_USER_PROXY ] ; then From 49e054aa81142c9bb914631a91d2a7375869e49d Mon Sep 17 00:00:00 2001 From: bockjoo Date: Tue, 7 Sep 2021 07:47:16 -0400 Subject: [PATCH 074/192] Remove distinction between the cached and the non-cached --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index b66266a..1ae3924 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -1027,8 +1027,11 @@ def updateXrdInfo (thesite) : if 'timeout' in xrootd_role or 'FATAL' in xrootd_role : #print ( "( endpont )",endpoint ) xrootd_role = XrdVersions[endpoint]['Role'] - sites[thesite]['xrootd_role'][xrd_idxes[idx]] = '['+xrootd_role+']' - if 'timeout' in xrootd_version or 'FATAL' in xrootd_version : sites[thesite]['xrootd_version'][xrd_idxes[idx]] = '['+XrdVersions[endpoint]['Version']+']' + #sites[thesite]['xrootd_role'][xrd_idxes[idx]] = '['+xrootd_role+']' + #if 'timeout' in xrootd_version or 'FATAL' in xrootd_version : sites[thesite]['xrootd_version'][xrd_idxes[idx]] = '['+XrdVersions[endpoint]['Version']+']' + sites[thesite]['xrootd_role'][xrd_idxes[idx]] = xrootd_role + if 'timeout' in xrootd_version or 'FATAL' in xrootd_version : sites[thesite]['xrootd_version'][xrd_idxes[idx]] = XrdVersions[endpoint]['Version'] + #def updateFederation (thesite) : # xrdInfo = sites[thesite] From f16fe215a95d801982e4515dce517ebebc199ce1 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 13 Sep 2021 15:42:12 -0400 Subject: [PATCH 075/192] Turn eveything to python3 --- .../XRDFED-kibana-probe_JSON_General.py | 146 +++++++++++------- .../uploadmetricGeneral.sh | 2 +- 2 files changed, 88 insertions(+), 60 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index a1a61f0..18aac14 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # functional probe and SLS extractor for the "federation" xroot services # highlights: # - stateless (i.e. run from cron whenever needed) @@ -26,8 +26,8 @@ import multiprocessing html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here #Bockjoo Uncomment and comment a line below -#LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' -LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe-general' +LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' +#LOCKFILE='/opt/TransferTeam/AAAOps/RedirectorServiceAvailability/var/lock/subsys/xrdfed-kibana-probe-general' # Bockjoo Uncomment and comment a line below #probes_json='KIBANA_PROBES.json' probes_json='KIBANA_PROBES_GENERAL.json' @@ -35,19 +35,20 @@ class Alarm(Exception): pass def alarm_handler(signum, frame): - print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") clear_lock() sys.exit(2) raise Alarm def clear_lock(): try: os.unlink(LOCKFILE) - except Exception,e: - print "could not remove lockfile:"+str(e) + except (Exception,e): + print ("could not remove lockfile:"+str(e)) def env_setup(): os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + #os.environ['X509_USER_PROXY']=os.environ['HOME']+'/.cmsuser.proxy' # DEBUG os.environ['KRB5CCNAME']='FILE:/dev/null' os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" def get_proxy(): @@ -62,12 +63,12 @@ def get_proxy(): def cleanup_proxy(): try: os.unlink(os.environ['X509_USER_PROXY']) - except Exception,e: - print "could not remove proxy file:"+str(e) + except (Exception,e): + print ("could not remove proxy file:"+str(e)) def try_lock(): ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) if ret > 0: - print "could not create lockfile" + print ("could not create lockfile") return False return True @@ -95,7 +96,8 @@ def prepare_dictionary(servicename,redirector): dic["xrdcp_below_time"] = 0 dic["xrdcp_above_time"] = 0 else: - (errtext,version,out) = xrd_info(redirector) + # OLD (errtext,version,out) = xrd_info(redirector) + (errtext,version,out) = xrd_info(redirector,'version') # NEW if(errtext): dic['version'] = 'unavailable' dic['status'] = 'unavailable' @@ -114,47 +116,66 @@ def xrdcp_test(redirector,file): "-DIReadCacheSize","0", "-DIRedirCntTimeout","180", "root://"+redirector+'/'+file, - '/dev/null']) + '/dev/null'],180) return (errtext,err,elapsed) -def xrd_info(redirector): - version = "(unknown)" - (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", - [redirector, - "query","config", # 1:kXR_QStats - "version"]) # a_ll stats - if not out: - errtext = '' - os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") - os.system("head -n 1 /root/aux.txt > /root/aux2.txt") - f = open('/root/aux2.txt', 'r') - version = f.read() - if not version: - version = "(unknown)" - else: - version = version[:-1] - else: - if not errtext: - try: - dom = xml.dom.minidom.parseString(out) - root_node = dom.documentElement - if root_node.tagName == 'statistics': - v_attr = root_node.getAttributeNode('ver') - version = v_attr.nodeValue - except Exception,e: - errtext = "ERROR: cannot parse answer:"+str(e) - return (errtext,version,out) -def run_xrd_commands(cmd,args): +def xrd_info(redirector,what): + theargs = [redirector, "query", "config", what] + if 'version' in what or 'role' in what : command = 'xrdfs' + else : + command = 'xrdmapc' + theargs = [ '--list', 'all', redirector ] + config_out = "unknown" + count = 0 + count_limit = 3 + timelimit = 2 + if 'xrdmapc' in command : + timelimit = 90 + count_limit = 1 + while (count < count_limit): + (errtext,out,err,elapsed) = run_xrd_commands(command, theargs, timelimit) #"xrdfs", + #[redirector, + # "query","config", # 1:kXR_QStats + # what]) # a_ll stats + + if 'xrdmapc' in command : + if out : + errtext = err + config_out = out + break + if err: + #errtext = '' + #config_out = 'error' + config_out = err.replace(b'\n',b'').decode() + else: + #config_out=out + #if 'xrdmapc' in command : + # break + #else : + config_out = out.replace(b'\n',b'').decode() + # # to check if count matters config_out=config_out+'+'+str(count) + if out.replace(b'\n',b'') : break + time.sleep(1) + count += 1 + #print ( 'DEBUG count = ',count,' config_out ',config_out, ' redirector ',redirector) + if not 'xrdmapc' in command : + if not config_out : config_out = 'timeout' + if 'Auth failed' in config_out : config_out = 'Auth failed' + return (errtext,config_out,out) + +def run_xrd_commands(cmd,args,timelimit): dev_null = open('/dev/null', 'r') errtxt = '' elapsed = -1.0 - xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command - "-DIConnectTimeout","30", - "-DITransactionTimeout","60", - "-DIRequestTimeout","60" ] + args - err = '' - out = '' + #xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + # "-DIConnectTimeout","30", + # "-DITransactionTimeout","60", + # "-DIRequestTimeout","60" ] + args + xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + #xrd_args = [ 'perl','-e',"alarm 30 ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + ] + args + #if 'xrdmapc' in cmd : + # xrd_args = [ cmd, ] + args try: - ran_try = True start = time.time() proc = subprocess.Popen(xrd_args, stdin=dev_null, @@ -163,22 +184,28 @@ def run_xrd_commands(cmd,args): (out, err) = proc.communicate() ret = proc.returncode elapsed = (time.time() - start) - err_redir_index = err.rfind('Received redirection to') - err_index3010 = err.rfind('(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK - err_index3005 = err.rfind('(error code: 3005') # (no user mapping) - INFN + #print ( ' out ',out ) + #if 'xrdmapc' in cmd : return ('',out,err,elapsed) + err_redir_index = err.rfind(b'Received redirection to') + err_index3010 = err.rfind(b'(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind(b'(error code: 3005') # (no user mapping) - INFN if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): errtxt = '' else: if(ret > 0): - errtxt = "client-side error - exit code "+str(ret)+"\n" - err_index = err.rfind('Last server error') + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind(b'Last server error') if err_index >= 0: - err_end_index=err.find("\n",err_index) - errtxt = errtxt + err[err_index:err_end_index] - except Exception,e: + err_end_index=err.find(b"\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except (Exception,e) : errtext = errtxt + "Exception: "+str(e) + out = 'Try did not work :O' + print(out) dev_null.close() + return (errtxt,out,err,elapsed) + def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): servicename=servicename.upper() notes_text = "Redirector: "+redirector @@ -234,14 +261,15 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= with open(html_dir + probes_json, 'a') as f: json.dump(dicci, f) f.write('\n') + def main(): debug = 0 atexit.register(clear_lock) if len(sys.argv) > 1: - if sys.argv[1] == '-d': - debug=1 + if sys.argv[1] == '-d': + debug=1 if not try_lock(): - sys.exit(1) + sys.exit(1) if not os.path.exists(html_dir): os.makedirs(html_dir) env_setup() @@ -330,10 +358,10 @@ def main(): #os.system('source ~/single_quotes.sh') #print(diccionaries) except Alarm: - print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") clear_lock() sys.exit(2) - print "Check ",html_dir+probes_json + print ("Check ",html_dir+probes_json) signal.alarm(0) if __name__ == '__main__': for file in os.listdir(html_dir): diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index f6958b6..caa9fc1 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -1,4 +1,4 @@ -#bin/sh +#!/bin/bash notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch From c7a5112b49f601f01f797ee09d714ff4d99b1a39 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 13 Sep 2021 15:47:57 -0400 Subject: [PATCH 076/192] Turn eveything to python3 --- AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index caa9fc1..e461618 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -14,7 +14,7 @@ thelog=$(pwd)/logs/uploadmetricGeneral.log # General script date echo INFO executing XRDFED-kibana-probe_JSON_General.py -python XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 +python3 XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 status=$? echo INFO $logs/XRDFED_probe_json.log cat $logs/XRDFED_probe_json.log From 54ee682012d0ee8e171d5cea6fded9228251189d Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 13 Sep 2021 19:08:57 -0400 Subject: [PATCH 077/192] Add the capability to test at Florida --- .../XRDFED-kibana-probe_JSON_General.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 18aac14..34131d2 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -27,7 +27,8 @@ html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here #Bockjoo Uncomment and comment a line below LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' -#LOCKFILE='/opt/TransferTeam/AAAOps/RedirectorServiceAvailability/var/lock/subsys/xrdfed-kibana-probe-general' +if 'ufhpc' in socket.gethostname() : + LOCKFILE='/opt/TransferTeam/AAAOps/RedirectorServiceAvailability/var/lock/subsys/xrdfed-kibana-probe-general' # Bockjoo Uncomment and comment a line below #probes_json='KIBANA_PROBES.json' probes_json='KIBANA_PROBES_GENERAL.json' @@ -48,7 +49,7 @@ def env_setup(): os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' - #os.environ['X509_USER_PROXY']=os.environ['HOME']+'/.cmsuser.proxy' # DEBUG + if 'ufhpc' in socket.gethostname() : os.environ['X509_USER_PROXY']=os.environ['HOME']+'/.cmsuser.proxy' os.environ['KRB5CCNAME']='FILE:/dev/null' os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" def get_proxy(): @@ -264,12 +265,14 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= def main(): debug = 0 - atexit.register(clear_lock) + if not 'ufhpc' in socket.gethostname(): + atexit.register(clear_lock) if len(sys.argv) > 1: if sys.argv[1] == '-d': debug=1 - if not try_lock(): - sys.exit(1) + if not 'ufhpc' in socket.gethostname(): + if not try_lock(): + sys.exit(1) if not os.path.exists(html_dir): os.makedirs(html_dir) env_setup() From 17ad04f9a0dd5c2e04a8969cca829171d17866b2 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 13 Sep 2021 19:19:36 -0400 Subject: [PATCH 078/192] Add the capability to test at Florida --- .../XRDFED-kibana-probe_JSON_General.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 34131d2..2dea931 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -43,7 +43,7 @@ def alarm_handler(signum, frame): def clear_lock(): try: os.unlink(LOCKFILE) - except (Exception,e): + except Exception as e: print ("could not remove lockfile:"+str(e)) def env_setup(): os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' @@ -64,7 +64,7 @@ def get_proxy(): def cleanup_proxy(): try: os.unlink(os.environ['X509_USER_PROXY']) - except (Exception,e): + except Exception as e: print ("could not remove proxy file:"+str(e)) def try_lock(): ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) @@ -199,7 +199,7 @@ def run_xrd_commands(cmd,args,timelimit): if err_index >= 0: err_end_index=err.find(b"\n",err_index) errtxt = errtxt + err[err_index:err_end_index] - except (Exception,e) : + except Exception as e : errtext = errtxt + "Exception: "+str(e) out = 'Try did not work :O' print(out) From 44735eb50a87493e01c5245657de5102d062b479 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 17 Sep 2021 23:41:59 -0400 Subject: [PATCH 079/192] Fix the xrdcp options --- .../XRDFED-kibana-probe_JSON_General.py | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 2dea931..43b4d8b 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -118,6 +118,7 @@ def xrdcp_test(redirector,file): "-DIRedirCntTimeout","180", "root://"+redirector+'/'+file, '/dev/null'],180) + #print ("DEBUG xrd_test ",redirector," errtext ",errtext, " err ", err ) return (errtext,err,elapsed) def xrd_info(redirector,what): theargs = [redirector, "query", "config", what] @@ -167,13 +168,15 @@ def run_xrd_commands(cmd,args,timelimit): dev_null = open('/dev/null', 'r') errtxt = '' elapsed = -1.0 - #xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command - # "-DIConnectTimeout","30", - # "-DITransactionTimeout","60", - # "-DIRequestTimeout","60" ] + args - xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command - #xrd_args = [ 'perl','-e',"alarm 30 ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command - ] + args + if 'xrdcp' in cmd : + xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + "-DIConnectTimeout","30", + "-DITransactionTimeout","60", + "-DIRequestTimeout","60" ] + args + else : + xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd ] + args + + #if 'xrdmapc' in cmd : # xrd_args = [ cmd, ] + args try: @@ -191,14 +194,17 @@ def run_xrd_commands(cmd,args,timelimit): err_index3010 = err.rfind(b'(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK err_index3005 = err.rfind(b'(error code: 3005') # (no user mapping) - INFN if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + #print ("errtxt will be empty because ",err) errtxt = '' else: + #print ("errtxt will not be empty because ret ",ret," err ",err) if(ret > 0): errtxt = "client-side error - exit code "+str(ret)+"\n" err_index = err.rfind(b'Last server error') if err_index >= 0: err_end_index=err.find(b"\n",err_index) errtxt = errtxt + err[err_index:err_end_index] + #print ("errtxt will not be empty because ret ",ret," err_index ", err_index, " err ",err) except Exception as e : errtext = errtxt + "Exception: "+str(e) out = 'Try did not work :O' @@ -225,11 +231,12 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= if (file_below): notes_text = notes_text + "File below: " + file_below (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) + #print ("DEBUG FILE_BELOW Test redirector ",redirector," err_below ",err_below) if err_below: availability = 'Degraded' #availinfo=availinfo+" Error below redirector "+err_below - dump_sane = re.sub('---*','__',dump_below) - c = c+"Error for file BELOW: "+err_below+". Dumpsane: "+dump_sane+ '.' + dump_sane = re.sub(b'---*',b'__',dump_below) + c = c+"Error for file BELOW: "+err_below+". Dumpsane: "+str(dump_sane.decode())+ '.' dicci['xrdcp_below_time'] = 0 else: #availinfo=availinfo+" File below: OK " @@ -240,11 +247,12 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= if(file_above): notes_text = notes_text + "File elsewhere: " + file_above (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) + #print ("DEBUG FILE_ABOVE Test redirector ",redirector," err_above ",err_above) if err_above : availability = 'Degraded' #availinfo=availinfo+" Error above redirector "+err_above - dump_sane = re.sub('---*','__',dump_above) - c = c+"Error for file ABOVE: "+err_above+". Dumpsane: "+dump_sane+'.' + dump_sane = re.sub(b'---*',b'__',dump_above) + c = c+"Error for file ABOVE: "+err_above+". Dumpsane: "+str(dump_sane.decode())+'.' dicci['xrdcp_above_time'] = 0 else: #availinfo = availinfo+" File above: OK " @@ -353,7 +361,7 @@ def main(): argus = services[service] argus['dicci'] = dicci if debug: - test_redirector(** services[xrd]) + test_redirector(** argus) else: t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" t.start() From 82abc08fb59edfc7ac6eb1c1e40a4ede489c44f8 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 1 Oct 2021 13:40:02 -0400 Subject: [PATCH 080/192] debug N/A --- .../XRDFED-kibana-probe_JSON_General.py | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 43b4d8b..72a716a 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -262,6 +262,7 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= dicci['xrdcp_above_time'] = 0 #availinfo = availinfo + " " + notes_text dicci['status']= str(availability) + print ("DEBUG redirector ",redirector, " dicci status ",dicci['status']) if c == '': c = 'N/A' c = c.replace("\n", "") From 0b7a942e5ec700b3fc0ff48b344f647f7b4f33c8 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 1 Oct 2021 14:11:57 -0400 Subject: [PATCH 081/192] debug N/A --- .../XRDFED-kibana-probe_JSON_General.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 72a716a..1f2ddf4 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -263,8 +263,8 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= #availinfo = availinfo + " " + notes_text dicci['status']= str(availability) print ("DEBUG redirector ",redirector, " dicci status ",dicci['status']) - if c == '': - c = 'N/A' + #if c == '': + # c = 'N/A' c = c.replace("\n", "") c = c.replace("\r", "") dicci ['Comment'] = c From 05c852092a4b13597b252ef886e7260ceca6a53a Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 1 Oct 2021 16:23:06 -0400 Subject: [PATCH 082/192] debug N/A --- .../XRDFED-kibana-probe_JSON_General.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 1f2ddf4..a931912 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -263,8 +263,10 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= #availinfo = availinfo + " " + notes_text dicci['status']= str(availability) print ("DEBUG redirector ",redirector, " dicci status ",dicci['status']) - #if c == '': - # c = 'N/A' + #01OCT2021 if c == '': + #01OCT2021 c = 'N/A' + if c == '': + c = 'No Comments' c = c.replace("\n", "") c = c.replace("\r", "") dicci ['Comment'] = c From a0479da19439bc5acd6cdce5e605091db6068aac Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 1 Oct 2021 17:34:04 -0400 Subject: [PATCH 083/192] sending the previous /var/www/html/aaa-probe/KIBANA_PROBES_GENERAL.json so that we can have a regular 15th minute entry for Grafana --- AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index e461618..615a572 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -13,6 +13,12 @@ thelog=$(pwd)/logs/uploadmetricGeneral.log # General script date +echo INFO sending the previous /var/www/html/aaa-probe/KIBANA_PROBES_GENERAL.json so that we can have a regular 15th minute entry for Grafana +python3 send_metrics.py > $logs/XRDFED_send.log 2>&1 +echo INFO content of logs/XRDFED_send.log +cat $logs/XRDFED_send.log +date + echo INFO executing XRDFED-kibana-probe_JSON_General.py python3 XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 status=$? From 8e79fa7ecce409ccaef481c5b5846ee6b54cfa7e Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 6 Oct 2021 13:53:16 -0400 Subject: [PATCH 084/192] Fix issue with {'error': 'list indices must be integers or slices, not str'} when parsing storage.json --- .../FedProbeSendAAAMetrics/create_fedmaps.py | 54 +++++++++++++------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index 1ae3924..a95aebf 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -275,7 +275,7 @@ def getSites(): ret[siteName]['xrootd_servers'] = [] ret[siteName]['xrootd_storage'] = [] ret[siteName]['federation'] = [] - #print ('getSites() ret keys ',ret.keys() ) + #print ('DEBUG getSites() ret keys ',ret.keys() ) #print ('getSites() ret["T0_CH_CERN"] ',ret["T0_CH_CERN"]) name = '' contact = '' @@ -288,7 +288,7 @@ def getSites(): break if not siteName: continue - #print ('getSites() site ',siteName) + #print ('DEBUG getSites() site ',siteName) services = site.findall('.//service') #ret[siteName]['name'].append(site.attrib['name']) if site.attrib['name'] : name = site.attrib['name'] @@ -301,7 +301,7 @@ def getSites(): contact = siteName #if site.attrib['contact'] : contact = site.attrib['contact'] - #print ('getSites() site contact ',) + #print ('DEBUG getSites() site contact ',) for service in services: flavor = service.attrib['flavour'] try: @@ -316,7 +316,7 @@ def getSites(): # (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") serviceName = service.attrib['hostname'] - #print ('getSites() services siteName ',siteName) + #print ('DEBUG getSites() services siteName ',siteName) ret[siteName]['sites'].append(siteName) ret[siteName]['hosts'].append(serviceName) ret[siteName]['flavors'].append(flavor) @@ -324,9 +324,9 @@ def getSites(): ret[siteName]['xrootd_version'].append(xrootd_version) ret[siteName]['xrootd_role'].append(xrootd_role) ret[siteName]['xrootd_servers'].append('_^_') - #print ('getSites() getStorageFromStorageJson siteName ',siteName) + #print ('DEBUG getSites() getStorageFromStorageJson siteName ',siteName) xrootd_storage = getStorageFromStorageJson ( siteName) - #print ('getSites() xrootd_storage ',xrootd_storage) + #print ('DEBUG getSites() xrootd_storage ',xrootd_storage) #if siteName in SiteStorages : print (' found it ') #else : print ('not found ') if siteName in SiteStorages : @@ -336,12 +336,12 @@ def getSites(): #print ( 'B ') xrootd_storage = getStorageFromStorageJson ( siteName ) updateSiteStorage ( siteName , xrootd_storage ) - #print ('getSites() done sitestorage ') + #print ('DEBUG getSites() done sitestorage ') ret[siteName]['xrootd_storage'].append(xrootd_storage) ret[siteName]['name'].append(name) ret[siteName]['contact'].append(contact) ret[siteName]['federation'].append('nowhere') - #print ('getSites() done ') + #print ('DEBUG getSites() done ') return ret @@ -352,20 +352,42 @@ def getStorageFromStorageJson ( site ): with open('/cvmfs/cms.cern.ch/SITECONF/'+site+'/storage.json') as f: storage = f.read() storage = json.loads(str(storage)) xrootd_volume = 'info_missing' - for vol in storage : + #print ('DEBUG getStorageFromStorageJson ( site ) before for vol in storage = ',storage) + #for line in xrdmapc_output_prod.splitlines() : + # line = line.decode() # python3 + #for vol in storage : + for index in range(len(storage)): + vol=storage[index] volume = str(vol['volume']) # ' '.join([str(elem) for elem in str(vol['volume']).split('_')]) + #print ('DEBUG getStorageFromStorageJson ( site ) volume = ',volume) if 'FEDERATION' in volume.upper() : continue - for proto in vol['protocols'] : - #print ( 'storage volume ',volume,proto['protocol'] ) - if 'XRootD' in str(proto['protocol']) : xrootd_volume = volume + #print ('DEBUG getStorageFromStorageJson ( site ) FEDERATION not in volume ',volume, ' vol protocols len ',len(vol['protocols']),' vol ',vol['protocols']) + #for proto in vol['protocols'] : + for index_proto in range(len(vol['protocols'])) : + proto=vol['protocols'][index_proto] + #print ( 'DEBUG getStorageFromStorageJson ( site ) storage index ', index_proto, ' volume ',volume, ' type ',type(proto) ) + #print ( 'DEBUG getStorageFromStorageJson ( site ) storage index ', index_proto, ' volume ',volume, proto['protocol'] ) + try : + if 'XRootD' in str(proto['protocol']) : xrootd_volume = volume + except : + continue + + #print ('DEBUG getStorageFromStorageJson ( site ) after for vol in storage') if 'info_missing' in xrootd_volume : - for vol in storage : + #for vol in storage : + for index in range(len(storage)): + vol=storage[index] volume = str(vol['volume']) # ' '.join([str(elem) for elem in str(vol['volume']).split('_')]) if 'FEDERATION' in volume.upper() : continue - for proto in vol['protocols'] : + #for proto in vol['protocols'] : + for index_proto in range(len(vol['protocols'])) : + proto=vol['protocols'][index_proto] #print ( 'storage volume ',volume,proto['protocol'] ) - if 'WebDAV' in str(proto['protocol']) : xrootd_volume = volume - #print ( site, xrootd_volume ) + try : + if 'WebDAV' in str(proto['protocol']) : xrootd_volume = volume + except : + continue + #print ('DEBUG getStorageFromStorageJson ( site ) site volume ', site, xrootd_volume ) # 3 ( #print ( 'storage volume ', len(storage), (str(storage[0]['volume'])).split('_')[1:]) #print ( 'storage volume ', len(storage), storage[1]) From 6d30e82cf0a783d741ba3548a6a64fe44a511969 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 15 Oct 2021 15:38:49 -0400 Subject: [PATCH 085/192] Fix mail option issue On the machine where this is running, the additional option gets scrambled. --- AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index 615a572..5fc435d 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -1,6 +1,6 @@ #!/bin/bash -notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch +notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch,bockjoo@gmailNOSPAMPLEASE.com DATE=$(date) echo $DATE @@ -36,7 +36,7 @@ if [ -f $thelog ] ; then a=1 [ $status -eq 0 ] && { grep -q -i "caught overall timeout" $logs/XRDFED_probe_json.log ; [ $(expr $a + $?) -eq $a ] && status=1 ; } ; if [ $status -ne 0 ] ; then - printf "$(/bin/hostname) $(basename $0)\n$(date)\n$(ls -al $thelog )\n$(cat $thelog)\n" | mail -s "$(/bin/hostname) $(basename $thelog)" $(echo $notifytowhom | sed 's#NOSPAMPLEASE##') -a $thelog + printf "$(/bin/hostname) $(basename $0)\n$(date)\n$(ls -al $thelog )\n$(cat $thelog)\n" | mail -s "$(/bin/hostname) $(basename $thelog)" $(echo $notifytowhom | sed 's#NOSPAMPLEASE##g') fi fi From a964e269ff959d807d06f709e22ec717fb14d14c Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 5 Nov 2021 00:43:07 -0400 Subject: [PATCH 086/192] Add an alert for non-zero exit at create_fedmaps.py --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index f1663ce..cbfced2 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -20,7 +20,9 @@ notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') [ -d $THEPATH/out ] || mkdir -p $THEPATH/out python3 $THEPATH/create_fedmaps.py > $THEPATH/create_fedmaps.log 2>&1 - +if [ $? -ne 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 $THEPATH/create_fedmaps.py\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom +fi if [ ! -r $FED_json ]; then echo "We have a problem creating JSON file.\n" printf "$(/bin/hostname -s) $(basename $0) We have a problem creating JSON file.\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom From 8e074083ff7b24b72f54612bd850b5811676126e Mon Sep 17 00:00:00 2001 From: bockjoo Date: Sun, 19 Dec 2021 15:52:46 -0500 Subject: [PATCH 087/192] Test --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index a95aebf..a322cd3 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -2,7 +2,7 @@ except ImportError: from elementtree import ElementTree as ET try: import json except ImportError: import simplejson as json - +# import sys #import urllib2, httplib, sys #import httplib, sys From d0b4135dcf7ca721c9b84158a8ee00a858cb649b Mon Sep 17 00:00:00 2001 From: root Date: Sun, 19 Dec 2021 22:12:12 +0100 Subject: [PATCH 088/192] sync --- .../XRDFED-kibana-probe_JSON_General.py | 1 + .../logs/XRDFED_probe_json.log | 15 --------------- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index a931912..9e91a75 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -49,6 +49,7 @@ def env_setup(): os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + #os.environ['X509_USER_PROXY']='/root/.globus/.proxy' if 'ufhpc' in socket.gethostname() : os.environ['X509_USER_PROXY']=os.environ['HOME']+'/.cmsuser.proxy' os.environ['KRB5CCNAME']='FILE:/dev/null' os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" diff --git a/AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log b/AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log deleted file mode 100644 index 1571b27..0000000 --- a/AAAOps/RedirectorServiceAvailability/logs/XRDFED_probe_json.log +++ /dev/null @@ -1,15 +0,0 @@ -('XRDFED_CMS-EU-BARI', ' is up ', True) -('XRDFED_CMS-TRANSIT01', ' is up ', True) -('XRDFED_CMS-US-FNAL', ' is up ', True) -('XRDFED_CMS-GLOBAL01', ' is up ', True) -('XRDFED_CMS-GLOBAL02', ' is up ', True) -('XRDFED_CMS-TRANSIT', ' is up ', True) -('XRDFED_CMS-US-UNL', ' is up ', True) -('XRDFED_CMS-EU-PISA', ' is up ', True) -('XRDFED_CMS-US', ' is up ', True) -('XRDFED_CMS-EU-LLR', ' is up ', True) -('XRDFED_CMS-EU-IPv6', ' is up ', True) -('XRDFED_CMS-EU', ' is up ', True) -('XRDFED_CMS-GLOBAL', ' is up ', True) -('XRDFED_CMS-TRANSIT02', ' is up ', True) -Check /var/www/html/aaa-probe/KIBANA_PROBES_GENERAL.json From 0ad64d71e0bb36a3638cf145bc68fa71a0c73097 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Mon, 20 Dec 2021 00:48:22 -0500 Subject: [PATCH 089/192] handle str has no attribute decode --- .../FedProbeSendAAAMetrics/create_fedmaps.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index a322cd3..d612093 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -2,7 +2,7 @@ except ImportError: from elementtree import ElementTree as ET try: import json except ImportError: import simplejson as json -# + import sys #import urllib2, httplib, sys #import httplib, sys @@ -1010,19 +1010,36 @@ def queryXrdmapc (redirector) : (err_info,xrdmapc_output_tran,dump_info) = xrd_info(redirector,'') with open(THEPATH+'out/xrdmapc_tran.txt','w' ) as f: for line in xrdmapc_output_tran.splitlines() : - f.write(str(line.decode())) + #f.write(str(line.decode())) + if type (line) is str : + #print ( "1 str "+line ) + f.write(line) + else : + #print ( "1 no str "+str(line.decode()) ) + f.write(str(line.decode())) f.write('\n') elif 'global02' in redirector : (err_info,xrdmapc_output_prod_2,dump_info) = xrd_info(redirector,'') with open(THEPATH+'out/xrdmapc_prod_2.txt','w' ) as f: for line in xrdmapc_output_prod_2.splitlines() : - f.write(str(line.decode())) + if type (line) is str : + #print ( "2 str "+line ) + f.write(line) + else : + #print ( "2 no str "+str(line.decode()) ) + f.write(str(line.decode())) f.write('\n') else : (err_info,xrdmapc_output_prod,dump_info) = xrd_info(redirector,'') with open(THEPATH+'out/xrdmapc_prod_1.txt','w' ) as f: for line in xrdmapc_output_prod.splitlines() : - f.write(str(line.decode())) + #f.write(str(line.decode())) + if type (line) is str : + #print ( "3 str "+line ) + f.write(line) + else : + #print ( "3 no str "+str(line.decode()) ) + f.write(str(line.decode())) f.write('\n') From faa176fe1c16cda210a857f220b5a1827e4752de Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 21 Jan 2022 12:00:10 -0500 Subject: [PATCH 090/192] xrdmapc timeout to 180 and removed .decode() for str object --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index d612093..b4e1098 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -99,7 +99,7 @@ def xrd_info(redirector,what): count_limit = 3 timelimit = 2 if 'xrdmapc' in command : - timelimit = 90 + timelimit = 180 count_limit = 1 while (count < count_limit): (errtext,out,err,elapsed) = run_xrd_commands(command, theargs, timelimit) #"xrdfs", @@ -719,7 +719,7 @@ def getRegionalRedirectors (): count = 0 #for line in set(Lines) : for line in xrdmapc_output_prod.splitlines() : - line = line.decode() # python3 + if type (line) is not str : line = line.decode() # python3 #print("Line{}: {}".format(count, line.split())) entries=line.split() #if 'Man' in entries[1] : @@ -789,7 +789,7 @@ def getXrootdServers () : count = 0 for line in xrdmapc_output_prod.splitlines() : #print("Line{}: {}".format(count, line.split())) - line = line.decode() + if type (line) is not str : line = line.decode() entries=line.split() #print ( entries ) #if entries[0] == 'Srv' : @@ -916,7 +916,7 @@ def getTransitionalXrootds () : #return xrdServers #count = 0 for line in xrdmapc_output_tran.splitlines() : - line = line.decode() + if type (line) is not str : line = line.decode() #print("Line{}: {}".format(count, line.split())) entries=line.split() #print ( entries From a23246656105dfca13a3cf0c7d47f1b3375ce4e8 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 16 Mar 2022 13:38:20 -0400 Subject: [PATCH 091/192] Throw Exception for sites with a bad storage.json --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index b4e1098..5b4aa6f 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -350,7 +350,10 @@ def getStorageFromStorageJson ( site ): from os import path if not path.exists('/cvmfs/cms.cern.ch/SITECONF/'+site+'/storage.json') : return 'no_storage_json' with open('/cvmfs/cms.cern.ch/SITECONF/'+site+'/storage.json') as f: storage = f.read() - storage = json.loads(str(storage)) + try: + storage = json.loads(str(storage)) + except: + return 'storage_json_bad' xrootd_volume = 'info_missing' #print ('DEBUG getStorageFromStorageJson ( site ) before for vol in storage = ',storage) #for line in xrdmapc_output_prod.splitlines() : From 0d7b513e8b2cad38c3e2cf307a946aad5137eb4d Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 21 Jul 2022 19:49:03 -0400 Subject: [PATCH 092/192] role and GRAFANA_PAGE added --- .../XRDFED-kibana-probe_JSON_General.py | 16 +++++++++++++++- .../uploadmetricGeneral.sh | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 9e91a75..78d70ac 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -94,6 +94,7 @@ def prepare_dictionary(servicename,redirector): if is_up == False: dic['version'] = 'unavailable' dic['status'] = 'unavailable' + #dic['role'] = 'unavailable' dic['comment'] = "Unpingable host" dic["xrdcp_below_time"] = 0 dic["xrdcp_above_time"] = 0 @@ -110,6 +111,18 @@ def prepare_dictionary(servicename,redirector): dic["xrdcp_above_time"] = 0 else: dic['version'] = version + #(errtext,role,out) = xrd_info(redirector,'role') + #if(errtext): + # dic['role'] = 'unavailable' + # dic['status'] = 'unavailable' + # errtext = errtext.replace("'", "") + # errtext = errtext.replace('"', '') + # dic['comment'] = "Error getting info from redirector: "+errtext + # dic["xrdcp_below_time"] = 0 + # dic["xrdcp_above_time"] = 0 + #else: + # dic['role'] = role + return dic def xrdcp_test(redirector,file): (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", @@ -263,7 +276,8 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= dicci['xrdcp_above_time'] = 0 #availinfo = availinfo + " " + notes_text dicci['status']= str(availability) - print ("DEBUG redirector ",redirector, " dicci status ",dicci['status']) + #print ("DEBUG redirector ",redirector, " dicci status ",dicci['status'], dicci['role'],dicci['version']) + print ("DEBUG redirector ",redirector, " dicci status ",dicci['status'], dicci['version']) #01OCT2021 if c == '': #01OCT2021 c = 'N/A' if c == '': diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index 5fc435d..f6a4ece 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -1,6 +1,7 @@ #!/bin/bash notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch,bockjoo@gmailNOSPAMPLEASE.com +GRAFANA_PAGE="https://monit-grafana.cern.ch/d/serviceAvailability/overview-service-availability?orgId=11&var-category=All&from=now-24h&to=now-15m%2Fm" DATE=$(date) echo $DATE From 095bc7e8c2b4ac466600b97b48a0d93342c660ff Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 07:16:27 -0400 Subject: [PATCH 093/192] Add version role to the email --- .../probe_create_send_aaa_metrics.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index cbfced2..4023748 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -36,6 +36,20 @@ if [ $? -ne 0 ] ; then fi [ -d $(dirname $THELOG) ] || mkdir -p $(dirname $THELOG) +# Server Version List +( + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Subject: XRootD Version Role List" + echo "Content-Type: text/html" + echo "" + echo "" + echo "" + grep "\"sites\"\|\"endpoints\"\|version\|role" $FED_json | while read site ; do read endpoints ; read version ; read role ; site=$(echo $site | cut -d\" -f4) ; endpoints=$(echo $endpoints | cut -d\" -f4) ; version=$(echo $version | cut -d\" -f4) ; [ "x$version" == "xtimeout" ] && version=$(echo $(xrdfs $endpoints query config version | grep ^v)) ; role=$(echo $role | cut -d\" -f4) ; [ "x$role" == "xtimeout" ] && role=$(echo $(xrdfs $endpoints query config role)) ; echo "" ; done + echo "
SiteEndpointVersionRole
$site $endpoints $version $role
" + echo "" +) | /usr/sbin/sendmail -t + + rm -f $THEPATH/fed.json python3 $THEPATH/aaa_federation.py --amq $THEPATH/credentials.json > $THEPATH/aaa_federation.log 2>&1 status=$? From f97b99ac3ce8796b3a83b60485a82e3005047f11 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 08:05:35 -0400 Subject: [PATCH 094/192] Add version role to the email 2 --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 4023748..09be462 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -37,6 +37,7 @@ fi [ -d $(dirname $THELOG) ] || mkdir -p $(dirname $THELOG) # Server Version List +if [ ] ; then ( echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") echo "Subject: XRootD Version Role List" @@ -48,7 +49,7 @@ fi echo "" echo "" ) | /usr/sbin/sendmail -t - +fi # if [ ] ; then rm -f $THEPATH/fed.json python3 $THEPATH/aaa_federation.py --amq $THEPATH/credentials.json > $THEPATH/aaa_federation.log 2>&1 From c7c9fd58188f9e7930070ccab3db1ab5de31e399 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 10:41:39 -0400 Subject: [PATCH 095/192] Add version role to the email 3 --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 09be462..ac358cb 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -37,7 +37,7 @@ fi [ -d $(dirname $THELOG) ] || mkdir -p $(dirname $THELOG) # Server Version List -if [ ] ; then +#if [ ] ; then ( echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") echo "Subject: XRootD Version Role List" @@ -45,11 +45,11 @@ if [ ] ; then echo "" echo "" echo "" - grep "\"sites\"\|\"endpoints\"\|version\|role" $FED_json | while read site ; do read endpoints ; read version ; read role ; site=$(echo $site | cut -d\" -f4) ; endpoints=$(echo $endpoints | cut -d\" -f4) ; version=$(echo $version | cut -d\" -f4) ; [ "x$version" == "xtimeout" ] && version=$(echo $(xrdfs $endpoints query config version | grep ^v)) ; role=$(echo $role | cut -d\" -f4) ; [ "x$role" == "xtimeout" ] && role=$(echo $(xrdfs $endpoints query config role)) ; echo "" ; done + grep "\"sites\"\|\"endpoints\"\|version\|role" $FED_json | while read site ; do read endpoints ; read version ; read role ; site=$(echo $site | cut -d\" -f4) ; endpoints=$(echo $endpoints | cut -d\" -f4) ; version=$(echo $version | cut -d\" -f4) ; port=$(echo $endpoints | cut -d: -f2) ; [ $port -eq 0 ] && continue ; [ "x$version" == "xtimeout" ] && version=$(echo $(xrdfs $endpoints query config version | grep ^v)) ; role=$(echo $role | cut -d\" -f4) ; [ "x$role" == "xtimeout" ] && role=$(echo $(xrdfs $endpoints query config role)) ; echo "" ; done echo "
SiteEndpointVersionRole
$site $endpoints $version $role
$site $endpoints $version $role
" echo "" ) | /usr/sbin/sendmail -t -fi # if [ ] ; then +#fi # if [ ] ; then rm -f $THEPATH/fed.json python3 $THEPATH/aaa_federation.py --amq $THEPATH/credentials.json > $THEPATH/aaa_federation.log 2>&1 From fbb8b68c501c714e341372f8462cdb2d90c9cad8 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 11:25:10 -0400 Subject: [PATCH 096/192] Add version role to the email 4 --- .../probe_create_send_aaa_metrics.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index ac358cb..6d05835 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -39,13 +39,30 @@ fi # Server Version List #if [ ] ; then ( + timeout=30 echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") echo "Subject: XRootD Version Role List" echo "Content-Type: text/html" echo "" echo "" echo "" - grep "\"sites\"\|\"endpoints\"\|version\|role" $FED_json | while read site ; do read endpoints ; read version ; read role ; site=$(echo $site | cut -d\" -f4) ; endpoints=$(echo $endpoints | cut -d\" -f4) ; version=$(echo $version | cut -d\" -f4) ; port=$(echo $endpoints | cut -d: -f2) ; [ $port -eq 0 ] && continue ; [ "x$version" == "xtimeout" ] && version=$(echo $(xrdfs $endpoints query config version | grep ^v)) ; role=$(echo $role | cut -d\" -f4) ; [ "x$role" == "xtimeout" ] && role=$(echo $(xrdfs $endpoints query config role)) ; echo "" ; done + grep "\"sites\"\|\"endpoints\"\|version\|role" $FED_json | \ + while read site ; do + read endpoints + read version + read role + site=$(echo $site | cut -d\" -f4) + endpoints=$(echo $endpoints | cut -d\" -f4) + version=$(echo $version | cut -d\" -f4) + role=$(echo $role | cut -d\" -f4 + # port 0 is an invalid port + port=$(echo $endpoints | cut -d: -f2) ; [ $port -eq 0 ] && continue + [ "x$version" == "xtimeout" ] && version=$(echo $(perl -e "alarm $timeout ; exec @ARGV" xrdfs $endpoints query config version | grep "^v\|Alarm clock")) + [ "$version" == "Alarm clock" ] && version="$timeout(sec)TO" + [ "x$role" == "xtimeout" ] && role=$(echo $(perl -e "alarm $timeout ; exec @ARGV" xrdfs $endpoints query config role)) + [ "$role" == "Alarm clock" ] && role="$timeout(sec)TO" + echo "" + done echo "
SiteEndpointVersionRole
$site $endpoints $version $role
$site $endpoints $version $role
" echo "" ) | /usr/sbin/sendmail -t From 6297e4f4cbe67159bf12acaf0a26540ec0cc4e63 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 11:49:49 -0400 Subject: [PATCH 097/192] Add version role to the email 4 --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 6d05835..d58e46d 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -39,7 +39,7 @@ fi # Server Version List #if [ ] ; then ( - timeout=30 + timeout=10 echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") echo "Subject: XRootD Version Role List" echo "Content-Type: text/html" From 39f1f518f7ff925c95b43ca699aeca91582e7dcd Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 12:40:34 -0400 Subject: [PATCH 098/192] Add version role to the email 5 --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index d58e46d..3267503 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -54,7 +54,7 @@ fi site=$(echo $site | cut -d\" -f4) endpoints=$(echo $endpoints | cut -d\" -f4) version=$(echo $version | cut -d\" -f4) - role=$(echo $role | cut -d\" -f4 + role=$(echo $role | cut -d\" -f4) # port 0 is an invalid port port=$(echo $endpoints | cut -d: -f2) ; [ $port -eq 0 ] && continue [ "x$version" == "xtimeout" ] && version=$(echo $(perl -e "alarm $timeout ; exec @ARGV" xrdfs $endpoints query config version | grep "^v\|Alarm clock")) From 8fad37469b1e125618006f1c92929085ef1b6814 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 13:12:45 -0400 Subject: [PATCH 099/192] Add version role to the email 6 --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 3267503..ccc8878 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -61,7 +61,9 @@ fi [ "$version" == "Alarm clock" ] && version="$timeout(sec)TO" [ "x$role" == "xtimeout" ] && role=$(echo $(perl -e "alarm $timeout ; exec @ARGV" xrdfs $endpoints query config role)) [ "$role" == "Alarm clock" ] && role="$timeout(sec)TO" - echo "$site $endpoints $version $role " + bgcolor="yellow" + [ "x$version" == "xv5.4.3" ] || bgcolor="red" + echo "$site $endpoints $version $role " done echo "" echo "" From 73ad00344622928a33bce04660a11789cac6407a Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 13:14:35 -0400 Subject: [PATCH 100/192] Add version role to the email 7 --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index ccc8878..002cbf1 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -52,6 +52,7 @@ fi read version read role site=$(echo $site | cut -d\" -f4) + echo "$site" | grep -q T3 && continue endpoints=$(echo $endpoints | cut -d\" -f4) version=$(echo $version | cut -d\" -f4) role=$(echo $role | cut -d\" -f4) From 4e4f184b126b36b52679e867e919ae4f2db29c9a Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 22 Jul 2022 14:31:00 -0400 Subject: [PATCH 101/192] Add version role to the email 8 --- .../probe_create_send_aaa_metrics.sh | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 002cbf1..7ea139b 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -39,6 +39,22 @@ fi # Server Version List #if [ ] ; then ( +is_int() { + if [ $1 ] ; then + theval=$1 + if [ "x$theval" == "x" ] ; then + return 1 + else + if [ "x`echo $theval | egrep -v '^[0-9]+$'`" != "x" ] ; then + return 1 + else + return 0 + fi + fi + else + return 1 + fi +} timeout=10 echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") echo "Subject: XRootD Version Role List" @@ -63,8 +79,23 @@ fi [ "x$role" == "xtimeout" ] && role=$(echo $(perl -e "alarm $timeout ; exec @ARGV" xrdfs $endpoints query config role)) [ "$role" == "Alarm clock" ] && role="$timeout(sec)TO" bgcolor="yellow" - [ "x$version" == "xv5.4.3" ] || bgcolor="red" - echo "$site $endpoints $version $role " + a=$(echo "$version" | grep v | sed "s#[a-z]##g" | sed "s#[A-Z]##g" | cut -d. -f1) + [ "x$a" == "x" ] && a=1 + [ $(is_int $a ; echo $?) -eq 0 ] || a=1 + a=$(expr 100000 \* $a) + b=$(echo "$version" | grep v | sed "s#[a-z]##g" | sed "s#[A-Z]##g" | cut -d. -f2) + [ "x$b" == "x" ] && b=1 + [ $(is_int $b ; echo $?) -eq 0 ] || b=1 + b=$(expr 1000 \* $b) + c=$(echo "$version" | grep v | sed "s#[a-z]##g" | sed "s#[A-Z]##g" | cut -d. -f3) + [ "x$c" == "x" ] && c=1 + [ $(is_int $c ; echo $?) -eq 0 ] || c=1 + c=$(expr 10 \* $c) + expr $a + $b + $c 2>/dev/null 1>/dev/null && \ + [ $(expr $a + $b + $c) -lt 504000 ] && bgcolor="red" + #[ "x$version" == "xv5.4.3" ] || [ "x$version" == "xv5.4.2" ] || bgcolor="red" + + echo "$site $endpoints $version <"'!'"-- a=$a b=$b c=$c status=$? --> $role " done echo "" echo "" From 1b3c38d893946d38bd6657940daf21cea5ae7fbd Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 27 Jul 2022 21:21:46 -0400 Subject: [PATCH 102/192] Section Development added --- AAAOps/FedProbeSendAAAMetrics/README | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/README b/AAAOps/FedProbeSendAAAMetrics/README index f7432d2..4756103 100644 --- a/AAAOps/FedProbeSendAAAMetrics/README +++ b/AAAOps/FedProbeSendAAAMetrics/README @@ -39,3 +39,15 @@ git clone https://github.com/dmwm/CMSMonitoring.git crontab -l > crontab echo '14,29,44,59 * * * * /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh > /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/logs/probe_create_send_aaa_metrics.log 2>&1' >> crontab # Add the cron to the gitlab + +[5] Development +On cmsio3 as bockjoo +cd /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/ +git add create_fedmaps.py +git status +git commit -m "create_fedmaps.py is updated" +git push origin master + +On vocms036.cern.ch as root +cd /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/ +git pull origin master From 040a0fb3d264787f9bca2d84dd50f14fb3252eec Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 7 Sep 2022 12:55:34 -0400 Subject: [PATCH 103/192] create_fedmaps.py and probe_create_send_aaa_metrics.sh are updated --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 15 +++++++++++++-- .../probe_create_send_aaa_metrics.sh | 4 ++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index 5b4aa6f..d08a3d6 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -791,7 +791,7 @@ def getXrootdServers () : #return xrdServers count = 0 for line in xrdmapc_output_prod.splitlines() : - #print("Line{}: {}".format(count, line.split())) + print("Line{}: {}".format(count, line.split())) if type (line) is not str : line = line.decode() entries=line.split() #print ( entries ) @@ -808,9 +808,10 @@ def getXrootdServers () : if host in HostSitenames : site=HostSitenames[host] else : + if 'localhost' in host : continue # give up (site,domain) = findSitename(host) updateHostSitename ( host, site ) - #print ( host, site ) + #print ( " host and site ",host, site ) #print ( "DEBUG xrdServers[site]['endpoints'] ", xrdServers[site]['endpoints'] ) #if '[::ffff:144.16.111.9]:11001' in endpoint : # site = 'T2_IN_TIFR' @@ -829,6 +830,14 @@ def getXrootdServers () : #if '[::ffff:144.16.111.9]:11001' in endpoint : # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') #else : + #if 'localhost' in site: + # print (" site has localhost") + # xrdServers[site]={} #'endpoints':'','hosts':'','flavors':'','federation':''} + # xrdServers[site]['endpoints']=['localhost'] + # xrdServers[site]['hosts']=[host] + # xrdServers[site]['flavors']=['XROOTD'] + # xrdServers[site]['federation']=['prod'] + #else: if not endpoint in xrdServers[site]['endpoints'] : xrdServers[site]['endpoints'].append ( endpoint ) xrdServers[site]['hosts'].append ( host ) @@ -837,6 +846,7 @@ def getXrootdServers () : #print("Line{}: {}".format(count, line.split()), ' idx = ',idx, endpoint,site,xrdServers[site]['federation']) #count += 1 + #print ("Returning xrdServers") return xrdServers def getXrootdServersFromFile () : @@ -1059,6 +1069,7 @@ def updateXrdInfo (thesite) : (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") #xrdInfo['xrootd_role'][xrd_idxes[idx]] = xrootd_role sites[thesite]['xrootd_role'][xrd_idxes[idx]] = xrootd_role + #print ("DEBUG updateXrdInfo ",thesite, sites[thesite]['endpoints'], " xrootd_role ",xrootd_role, " xrootd_version ",xrootd_version) if not 'timeout' in xrootd_role and not 'FATAL' in xrootd_role : if not 'timeout' in xrootd_version and not 'FATAL' in xrootd_version : updatexrdVersions ( endpoint, xrootd_role, xrootd_version ) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 7ea139b..f7c3488 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -21,11 +21,11 @@ notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') python3 $THEPATH/create_fedmaps.py > $THEPATH/create_fedmaps.log 2>&1 if [ $? -ne 0 ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 $THEPATH/create_fedmaps.py\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 $THEPATH/create_fedmaps.py\n\n$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi if [ ! -r $FED_json ]; then echo "We have a problem creating JSON file.\n" - printf "$(/bin/hostname -s) $(basename $0) We have a problem creating JSON file.\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem creating JSON file.\n\nn$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom exit 1 fi From 522cd6a47e4dadb04c50566e8a49528d98e2bc8c Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 8 Sep 2022 14:01:15 -0400 Subject: [PATCH 104/192] uploadmetricGeneral.sh is updated --- .../uploadmetricGeneral.sh | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index f6a4ece..2276475 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -1,6 +1,8 @@ #!/bin/bash -notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch,bockjoo@gmailNOSPAMPLEASE.com +#notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch,bockjoo@gmailNOSPAMPLEASE.com +notifytowhom=bockjoo__AT__gmail__dot__com +notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') GRAFANA_PAGE="https://monit-grafana.cern.ch/d/serviceAvailability/overview-service-availability?orgId=11&var-category=All&from=now-24h&to=now-15m%2Fm" DATE=$(date) @@ -23,6 +25,26 @@ date echo INFO executing XRDFED-kibana-probe_JSON_General.py python3 XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 status=$? +if [ $status -ne 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 XRDFED-kibana-probe_JSON_General.py\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom +fi +rdirs=$(grep redirector $logs/XRDFED_probe_json.log | awk '{print $3}') +rdirs_degraded= +for r in $rdirs ; do + grep redirector $logs/XRDFED_probe_json.log | grep "$r" | grep -q Available # Degraded # Available + if [ $? -ne 0 ] ; then + #grep redirector $logs/XRDFED_probe_json.log | grep "$r" | grep -q Degraded # Available # Degraded + #if [ $? -eq 0 ] ; then + rdir_degraded=$(grep redirector $logs/XRDFED_probe_json.log | grep "$r" | awk '{print $(NF-1)"+"$NF}') + rdirs_degraded="$rdirs_degraded ${r}+$rdir_degraded" + #else + #3fi + fi +done +if [ "x$rdirs_degraded" != "x" ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom +fi + echo INFO $logs/XRDFED_probe_json.log cat $logs/XRDFED_probe_json.log date From 4d9b6d6f3b1aee1505f6ccf651e4cab253b7f7e7 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 8 Sep 2022 15:03:23 -0400 Subject: [PATCH 105/192] uploadmetricGeneral.sh is updated --- AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index 2276475..67175d6 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -42,7 +42,7 @@ for r in $rdirs ; do fi done if [ "x$rdirs_degraded" != "x" ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdcp -d 1 -f -DIReadCacheSize 0 -DIRedirCntTimeout 180 -DIConnectTimeout 30 -DITransactionTimeout 60 -DIRequestTimeout 60 root://rdir//store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root /dev/null\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi echo INFO $logs/XRDFED_probe_json.log From e64016de115ddbcae8679500655e737c2957be70 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 8 Sep 2022 15:11:25 -0400 Subject: [PATCH 106/192] XRDFED-kibana-probe_JSON_General.py is updated --- .../XRDFED-kibana-probe_JSON_General.py | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 78d70ac..c879dc1 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -285,6 +285,7 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= c = c.replace("\n", "") c = c.replace("\r", "") dicci ['Comment'] = c + print ("DEBUG rdir ", redirector," dicci['Comment'] ",dicci['Comment']) with open(html_dir + probes_json, 'a') as f: json.dump(dicci, f) f.write('\n') From dfd2ee66dbee4027964c348d06d4018e5b4a2d36 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 8 Sep 2022 17:00:58 -0400 Subject: [PATCH 107/192] XRDFED-kibana-probe_JSON_General.py is updated --- .../XRDFED-kibana-probe_JSON_General.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index c879dc1..2c13ebf 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -90,7 +90,7 @@ def check_ping(hostname): def prepare_dictionary(servicename,redirector): is_up = check_ping(redirector[:redirector.find(':')]) dic={'service':servicename, 'host': redirector[:redirector.find(':')]} - print(servicename," is up ",is_up) + print(servicename," redirector= ",redirector," is up ",is_up) if is_up == False: dic['version'] = 'unavailable' dic['status'] = 'unavailable' @@ -124,6 +124,10 @@ def prepare_dictionary(servicename,redirector): # dic['role'] = role return dic +def xrdmapc_test(redirector): + (errtext,out,err,elapsed) = run_xrd_commands("xrdmapc", ["--list","all", redirector], 180) + #print ("DEBUG xrd_test ",redirector," errtext ",errtext, " err ", err ) + return (errtext,err,elapsed) def xrdcp_test(redirector,file): (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", ["-d","1", @@ -285,7 +289,13 @@ def test_redirector(dicci, servicename, redirector, file_below=None, file_above= c = c.replace("\n", "") c = c.replace("\r", "") dicci ['Comment'] = c + print ("DEBUG rdir ", redirector," dicci['Comment'] ",dicci['Comment']) + + # xrdmapc test 08SEP2022 + (err_xrdmapc,dump_xrdmapc,elapsed_xrdmapc) = xrdmapc_test(redirector) + print ("DEBUG rdir ", redirector, " xrdmapc err ",err_xrdmapc, " dump ",dump_xrdmapc, " elapsed ",elapsed_xrdmapc) + with open(html_dir + probes_json, 'a') as f: json.dump(dicci, f) f.write('\n') From d6a85f64569bc7e6cc62d40ae1585890803bc524 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 8 Sep 2022 18:21:04 -0400 Subject: [PATCH 108/192] uploadmetricGeneral.sh is updated --- .../RedirectorServiceAvailability/uploadmetricGeneral.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index 67175d6..e3e5e6a 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -45,6 +45,13 @@ if [ "x$rdirs_degraded" != "x" ] ; then printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdcp -d 1 -f -DIReadCacheSize 0 -DIRedirCntTimeout 180 -DIConnectTimeout 30 -DITransactionTimeout 60 -DIRequestTimeout 60 root://rdir//store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root /dev/null\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi +# check the xrdmapc Operation expired error +for r in $rdirs ; do + grep xrdmapc $logs/XRDFED_probe_json.log | grep "$r" | grep -q "\[ERROR\] Operation expired" + if [ $? -eq 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have the Operation expired with $r\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdmapc --list all $r\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + fi +done echo INFO $logs/XRDFED_probe_json.log cat $logs/XRDFED_probe_json.log date From cd37e51116da259c773afad6caf4b8fd0c6d6640 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 9 Sep 2022 10:27:53 -0400 Subject: [PATCH 109/192] uploadmetricGeneral.sh is updated --- AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index e3e5e6a..5165af4 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -28,6 +28,8 @@ status=$? if [ $status -ne 0 ] ; then printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 XRDFED-kibana-probe_JSON_General.py\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi + +if [ ] ; then rdirs=$(grep redirector $logs/XRDFED_probe_json.log | awk '{print $3}') rdirs_degraded= for r in $rdirs ; do @@ -44,6 +46,7 @@ done if [ "x$rdirs_degraded" != "x" ] ; then printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdcp -d 1 -f -DIReadCacheSize 0 -DIRedirCntTimeout 180 -DIConnectTimeout 30 -DITransactionTimeout 60 -DIRequestTimeout 60 root://rdir//store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root /dev/null\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi +fi # check the xrdmapc Operation expired error for r in $rdirs ; do From c60c2300a824d8553dfa22455836fa8f75bdbcf5 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 9 Sep 2022 10:30:23 -0400 Subject: [PATCH 110/192] probe_create_send_aaa_metrics.sh is updated --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index f7c3488..24468d8 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -37,7 +37,7 @@ fi [ -d $(dirname $THELOG) ] || mkdir -p $(dirname $THELOG) # Server Version List -#if [ ] ; then +if [ ] ; then ( is_int() { if [ $1 ] ; then @@ -100,7 +100,7 @@ is_int() { echo "" echo "" ) | /usr/sbin/sendmail -t -#fi # if [ ] ; then +fi # if [ ] ; then rm -f $THEPATH/fed.json python3 $THEPATH/aaa_federation.py --amq $THEPATH/credentials.json > $THEPATH/aaa_federation.log 2>&1 From 7b36d3407ae85fd60db7416b502c68c72b6b114e Mon Sep 17 00:00:00 2001 From: bockjoo Date: Fri, 9 Sep 2022 11:10:41 -0400 Subject: [PATCH 111/192] README --- AAAOps/FedProbeSendAAAMetrics/README | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/README b/AAAOps/FedProbeSendAAAMetrics/README index 4756103..9668ab1 100644 --- a/AAAOps/FedProbeSendAAAMetrics/README +++ b/AAAOps/FedProbeSendAAAMetrics/README @@ -1,3 +1,8 @@ +[0] How to Check Things + 1 To check the hostname the ip from an xrdmapc belongs to, for example, run + +xrdfs [2001:e10:4000:122::105]:11001 query config sitename + [1] Essential Files and Directories create_fedmaps.py aaa_federation.py From 437eb8352cac6df95f2195f3238039ec8f912cc6 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 13 Oct 2022 01:04:04 -0400 Subject: [PATCH 112/192] create_fedmaps.py is updated --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index d08a3d6..8749841 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -791,7 +791,7 @@ def getXrootdServers () : #return xrdServers count = 0 for line in xrdmapc_output_prod.splitlines() : - print("Line{}: {}".format(count, line.split())) + #print("Line{}: {}".format(count, line.split())) if type (line) is not str : line = line.decode() entries=line.split() #print ( entries ) @@ -927,10 +927,10 @@ def getTransitionalXrootds () : xrdServers[site]['federation'] = [] xrdServers[site]['flavors'] = [] #return xrdServers - #count = 0 + count = 0 for line in xrdmapc_output_tran.splitlines() : if type (line) is not str : line = line.decode() - #print("Line{}: {}".format(count, line.split())) + #print("Line{}: {}".format(count, line.split())) entries=line.split() #print ( entries idx = 0 @@ -961,7 +961,7 @@ def getTransitionalXrootds () : xrdServers[site]['federation'].append ( 'trans' ) #print (' endpoint ',endpoint ) - #count += 1 + count += 1 return xrdServers def getXrootdsFromTransFile () : @@ -1220,6 +1220,7 @@ def getAllSiteRedirectors () : #xrdTranServers = getXrootdsFromTransFile() xrdTranServers = getTransitionalXrootds() #print ( 'DEBUG xrdTranServers ', xrdTranServers) + print ( 'DEBUG Doing sites') for site in sites.keys(): #print ( 'DEBUG Doing site g ',site ) if 'T0_CH_CERN' in site : @@ -1308,6 +1309,7 @@ def getAllSiteRedirectors () : #print ( 'Doing site x ',site, sites[site]) updateContactInfo() + print ("DEBUG return sites for getAllSiteRedirectors") return sites except Exception as e : err={} From d95c87f578b36edbc5eb8511bf1a61e9842cdd49 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 12:21:11 -0500 Subject: [PATCH 113/192] cms_sam3_check.sh is added --- .../FedProbeSendAAAMetrics/cms_sam3_check.sh | 923 ++++++++++++++++++ 1 file changed, 923 insertions(+) create mode 100755 AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh diff --git a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh new file mode 100755 index 0000000..5607e26 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh @@ -0,0 +1,923 @@ +#!/bin/bash +# +# Bockjoo Kim +# WHAT : Minimally large effort Emulation of CMS SAM Dashboard in plain cgi/HTML or for failed test alert +# Reference : https://monit-grafana.cern.ch/d/2LNN04NMk/user-carizapo-sitemon?orgId=11 +# https://monit-kibana-acc.cern.ch/kibana/app/kibana#/discover?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-17h,to:now))&_a=(columns:!(data.metric_name,data.service_flavour,data.dst_experiment_site,data.status),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'monit_prod_sam3_enr_*',key:data.dst_experiment_site,negate:!f,params:(query:T2_US_Florida),type:phrase,value:T2_US_Florida),query:(match:(data.dst_experiment_site:(query:T2_US_Florida,type:phrase))))),index:'monit_prod_sam3_enr_*',interval:auto,query:(language:kuery,query:''),sort:!(metadata.timestamp,desc)) +# https://monit-kibana-acc.cern.ch/kibana/goto/7f572448ab2bbd36280ad87a3f499abe +# SAM test twiki : https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsSAMTests +# SAM test twiki : https://twiki.cern.ch/twiki/bin/view/CMS/SAMTestEmulationHowto +# Gitlab : https://gitlab.cern.ch/etf/cmssam/-/tree/master +# +#inputs=/opt/cms/services/HammerCloudXrootdMonitoring +inputs=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +notifytowhom=bockjoo__AT__gmail__dot__com +#THESITE=T2_US_Florida +#tld=rc.ufl.edu # change this +#port=8443 +#webroot=http://$(/bin/hostname -s).${tld}:${port}/cgi-bin +#thesite= +#used_percent_warning=70 # change this if needed +#used_percent=$(df -h /$(echo $inputs | cut -d/ -f2) | grep -v Filesystem | awk '{print $(NF-1)}' | sed 's#%##g') +#if [ $used_percent -gt $used_percent_warning ] ; then +# ( +# echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") +# echo "Subject: Warning not enough space in /opt" +# echo "Content-Type: text/html" +# echo "" +# echo "
"
+#     echo "$(/bin/hostname)" "$0" "
"
+#     echo "Used: $used_percent (%) vs Used Warning: $used_percent_warning (%)"
+#     du -sh $inputs/*
+#     echo "
" +# echo "" +# #cat $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +# echo +# ) | /usr/sbin/sendmail -t +#fi + +basename_0=$(basename $0) +gte="now-40m/m" +lte="now" +scroll="20m" +size=10000 +theprofile=CMS_CRITICAL_FULL +#theprofile=CMS_CRITICAL +#theprofile=CMS_FULL +DBID=9673 +EVERY_X_HOUR=1 +#unique_f=data.metric_name # data.CRAB_JobLogURL +##tie_breaker_id=metadata.kafka_timestamp # data.CRAB_TaskCreationDate +#tie_breaker_id=metadata.timestamp # data.CRAB_TaskCreationDate +unique_f=metadata.timestamp # data.CRAB_JobLogURL +tie_breaker_id=data.metric_name # data.CRAB_TaskCreationDate +tie_breaker_id=metadata.kafka_timestamp # +#index_json="{\"search_type\": \"query_then_fetch\", \"index\": [\"monit_prod_cms_rucio_raw_events*\"], \"ignore_unavailable\": true}" + +detail_link_prefix="https://monit-grafana.cern.ch/d/siYq3DxZz/wlcg-sitemon-test-details?orgId=20&var-id=" +cms_wiki_how_to_sam="https://twiki.cern.ch/twiki/bin/view/CMS/SAMTestEmulationHowto" + +#printf "Content-type: text/html\n\n" + +functions=$inputs/`basename ${0}`.fuctions.`date -u +%s` +perl -n -e 'print if /^####### BEGIN Functions 12345/ .. /^####### ENDIN Functions 12345/' < $0 | grep -v "Functions 12345" > $functions +source $functions +rm -f $functions + +#[ $# -gt 0 ] && thesite=$1 +#[ $# -gt 1 ] && gte=$2 +#[ $# -gt 2 ] && lte=$3 + + +#thestrings=`echo $QUERY_STRING | sed "s#&# #g"` +#for thestring in $thestrings ; do +# echo "$thestring" | grep -q -e "test_howto\|test_id\|test_output\|site\|gte\|lte\|match_all" +# [ $? -eq 0 ] || continue +# eval $thestring +# if [ "x$site" != "x" ] ; then +# thesite=$site +# fi +#done + +#thestrings=`echo $QUERY_STRING | sed "s#&# #g"` +#for thestring in $thestrings ; do +# echo "$thestring" | grep -q -e "test_howto\|test_id\|test_output\|site\|gte\|lte\|match_all" +# [ $? -eq 0 ] || continue +# if [ "x$test_id" != "x" ] ; then +# print_test_output $test_id $site +# exit 0 +# fi +# if [ "x$test_howto" != "x" ] ; then +# echo "
"
+#       echo "How to emulate this SAM test ( $test_howto )"
+#       thefunction=readme_$(echo $test_howto | sed "s|\.|__dot__|g" | sed "s|-|__dash__|g")
+#       $thefunction
+#       echo "
" +# exit 0 +# fi +#done + +#if [ "x$thesite" == "x" ] ; then +# thesite=$(basename $(readlink -f /cvmfs/cms.cern.ch/SITECONF/local)) +#fi +#if [ "x$thesite" == "x" ] ; then +# printf "Content-type: text/html\n\n" +# echo ERROR $(basename $0) provide the argument for the site name +# exit 0 +#fi +#if [ $(echo $thesite | wc -w) -ne 1 ] ; then +# printf "Content-type: text/html\n\n" +# echo ERROR $(basename $0) Unfortunately number of sitename is not unique or zero. +# exit 0 +#fi + + +#now_is=$(date +%s)_${thesite} +#now_is=$(date +%s)_american_sites +#echo "" > $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + +# All the template json files needed here found in $inputs +#sitemon_agg_metric_json=es.q.match_sitemon_raw_metric.json +token_txt=cms_sam3_check_token.txt +sort_search_json=cms_sam3_check_sort_search.json +sort_search_after_json=cms_sam3_check_sort_search_after.json +search_details=es.q.match_sitemon_raw_metric_search_details.json +sort_search_id_json=es.q.match_sitemon_raw_metric_sort_search_id.json +perl -n -e 'print if /^####### BEGIN token_txt/ .. /^####### ENDIN token_txt/' < $0 | grep -v "token_txt" > $token_txt +perl -n -e 'print if /^####### BEGIN sort_search_json/ .. /^####### ENDIN sort_search_json/' < $0 | grep -v "sort_search_json" > ${sort_search_json}.in +perl -n -e 'print if /^####### BEGIN sort_search_after_json/ .. /^####### ENDIN sort_search_after_json/' < $0 | grep -v "sort_search_after_json" > ${sort_search_after_json}.in + +GTE=$(echo $gte | sed 's#/# #' | awk '{print $1}') +LTE=$(echo $lte | sed 's#/# #' | awk '{print $1}') +#sites="T2_US_Caltech T2_US_Florida T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Wisconsin T2_BR_SPRACE T2_BR_UERJ" +#sites=T2_US_Florida +#sites="T2_US_Florida T2_US_Caltech T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Wisconsin T2_BR_SPRACE T2_BR_UERJ T1_US_FNAL T2_US_Vanderbilt T2_UK_London_Brunel T3_US_Colorado" # T2_KR_KISTI T2_FR_CCIN2P3 T0_CH_CERN T1_UK_RAL" +#sites="T2_US_Florida T2_RU_ITEP" # T2_PK_NCP" # T2_US_Caltech T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Wisconsin T2_BR_SPRACE T2_BR_UERJ T1_US_FNAL T2_KR_KISTI T2_FR_CCIN2P3 T0_CH_CERN T1_UK_RAL" + +#sites="T2_US_Florida T2_AT_Vienna T2_BE_IIHE T2_BE_UCL T2_BR_SPRACE T2_BR_UERJ T2_CH_CERN T2_CH_CSCS T2_CN_Beijing T2_DE_DESY T2_DE_RWTH T2_EE_Estonia T2_ES_CIEMAT T2_ES_IFCA T2_FI_HIP T2_FR_GRIF_IRFU T2_FR_IPHC T2_GR_Ioannina T2_HU_Budapest T2_IN_TIFR T2_IT_Bari T2_IT_Legnaro T2_IT_Pisa T2_IT_Rome T2_KR_KISTI T2_PK_NCP T2_PL_Swierk T2_PT_NCG_Lisbon T2_RU_IHEP T2_RU_INR T2_RU_ITEP T2_RU_JINR T2_TR_METU T2_TW_NCHC T2_UA_KIPT T2_UK_London_Brunel T2_UK_London_IC T2_UK_SGrid_RALPP T2_US_Caltech T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Vanderbilt T2_US_Wisconsin T0_CH_CERN" +if [ $# -lt 1 ] ; then + echo ERROR: $(basename $0) site + exit 1 +fi + +[ $# -gt 0 ] && thesite="$1" + +error_message="" +#error_message_html="" +#extra_metric_names="" +#for thesite in $sites ; do + + +#sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" $inputs/${sitemon_agg_metric_json}.in > ${sitemon_agg_metric_json}_${now_is} +#nhits=$(curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${sitemon_agg_metric_json}_${now_is}" 2>/dev/null | sed 's#"hits":#\nhits_total #' | grep ^hits_total | cut -d: -f2 | cut -d, -f1) + +#sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_json}.in > ${sort_search_json}_${now_is} +#nsearch=$(expr $nhits / $size) + +#echo "
"
+#echo "  Site:$thesite SAM Test Status  "
+#echo "<center><FONT color='green'> <b> Site:$thesite SAM Test Status </b> </FONT></center>"
+#echo "
  Site:$thesite SAM Test Status  
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html
+#echo INFO Site=$thesite nhits=$nhits query json follows
+#cat ${sort_search_json}_${now_is}
+#echo "
" + +#for i in $(seq 0 $nsearch) ; do +i=0 +while : ; do + i=$(expr $i + 1) + if [ $i -eq 1 ] ; then + sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_json}.in > $inputs/${sort_search_json} + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out + rm -f $inputs/${sort_search_json} + else + sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s|@@search_after@@|$search_after|" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_after_json}.in > $inputs/${sort_search_after_json} + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_after_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out + rm -f $inputs/${sort_search_after_json} + fi + #echo DEBUG $inputs/$(basename $0 | sed "s#\.sh##").$i.out_${now_is} + entry=$(sed "s#\"_source\":#\n\"_source\":#g" $inputs/$(echo $basename_0 | sed "s#\.sh##").$i.out | grep ^\"_source\": | wc -l) + search_after=$(sed 's#sort":#\nsort":#g' $inputs/$(basename $0 | sed "s#\.sh##").$i.out | grep ^sort | tail -1 | cut -d\[ -f2- | cut -d\] -f1) + #echo DEBUG entry=$entry + #echo DEBUG search_after=$search_after + [ $entry -lt $size ] && break + [ "x$search_after" == "xnull,null" ] && break + [ "x$search_after" == "x" ] && break +done + +#exit 0 + +# if [ $i -eq 0 ] ; then +# curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${sort_search_json}_${now_is}" 2>/dev/null 1> $(basename $0 | sed "s#\.sh##").$i.out_${now_is} +# search_after=$(sed 's#sort":#\nsort":#g' $(basename $0 | sed "s#\.sh##").$i.out_${now_is} | grep ^sort | tail -1 | cut -d\[ -f2- | cut -d\] -f1) +# else +# sed -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s|@@search_after@@|$search_after|" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_after_json}.in > ${sort_search_after_json}_${now_is} +# curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${sort_search_after_json}_${now_is}" 2>/dev/null 1> $(basename $0 | sed "s#\.sh##").$i.out_${now_is} +# search_after=$(sed 's#sort":#\nsort":#g' $(basename $0 | sed "s#\.sh##").$i.out_${now_is} | grep ^sort | tail -1 | cut -d\[ -f2- | cut -d\] -f1) +# fi +#done + +#ls $inputs/$(basename $0 | sed "s#\.sh##").*.out_${now_is} +#exit 0 + +#timestamps=$(sed 's#"data":#\n"data":#g' $inputs/$(basename $0 | sed "s#\.sh##").*.out | sed 's#,"#,\n"#g' | grep "^\"_id\"\|^\"metric_name\"\|^\"dst_hostname\"\|^\"status\"\|^\"service_flavour\"\|^\"timestamp\"" | grep "\"timestamp\"" | cut -d: -f2 | cut -d, -f1 | cut -d\} -f1 | sort -rn | uniq) +#LEGEND=" +#01:org.cms.WN-squid:(/cms/Role_lcgadmin) +#02:org.cms.WN-frontier:(/cms/Role_lcgadmin) +#03:org.cms.WN-basic:(/cms/Role_lcgadmin) +#04:org.cms.SRM-GetPFNFromTFC:(/cms/Role_production) +#05:org.cms.WN-xrootd-access:(/cms/Role_lcgadmin) +#06:org.cms.SE-xrootd-connection +#07:org.cms.WN-xrootd-fallback:(/cms/Role_lcgadmin) +#08:org.cms.WN-mc:(/cms/Role_lcgadmin) +#09:org.cms.WN-env:(/cms/Role_lcgadmin) +#10:org.cms.WN-isolation:(/cms/Role_lcgadmin) +#11:org.cms.SRM-VOPut:(/cms/Role_production) +#12:org.sam.CONDOR-JobSubmit:(/cms/Role_lcgadmin) +#13:org.cms.WN-analysis:(/cms/Role_lcgadmin) +#14:org.cms.SE-xrootd-contain +#15:org.cms.SRM-VOGet:(/cms/Role_production) +#16:org.cms.SE-xrootd-version +#17:org.cms.SE-xrootd-read +#99:org.cms.WN-cvmfs:(/cms/Role_lcgadmin)" +LEGEND=" +01:org.cms.WN-squid:(/cms/Role_lcgadmin) +02:org.cms.WN-frontier:(/cms/Role_lcgadmin) +03:org.cms.WN-basic:(/cms/Role_lcgadmin) +04:org.cms.SRM-GetPFNFromTFC:(/cms/Role_production) +05:org.cms.WN-xrootd-access:(/cms/Role_lcgadmin) +06:org.cms.SE-xrootd-connection +07:org.cms.WN-xrootd-fallback:(/cms/Role_lcgadmin) +08:org.cms.WN-mc:(/cms/Role_lcgadmin) +09:org.cms.WN-env:(/cms/Role_lcgadmin) +10:org.cms.WN-isolation:(/cms/Role_lcgadmin) +11:org.cms.SRM-VOPut:(/cms/Role_production) +12:org.sam.CONDOR-JobSubmit:(/cms/Role_lcgadmin) +13:org.cms.WN-analysis:(/cms/Role_lcgadmin) +14:org.cms.SE-xrootd-contain +15:org.cms.SRM-VOGet:(/cms/Role_production) +16:org.cms.SE-xrootd-version +17:org.cms.SE-xrootd-read +18:org.cms.SE-WebDAV-1connection +19:org.cms.SE-WebDAV-2ssl +20:org.cms.SE-WebDAV-3extension +21:org.cms.SE-WebDAV-4crt-read +22:org.cms.SE-WebDAV-5open-access +23:org.cms.SE-WebDAV-6crt-write +24:org.cms.SE-WebDAV-7macaroon +25:org.cms.SE-WebDAV-9summary +26:org.cms.CONDOR-Ping:(/cms-ce-token) +99:org.cms.WN-cvmfs:(/cms/Role_lcgadmin)" +metrics=$(printf "$LEGEND\n" | cut -d: -f2 | grep ^org | sed "s#^org#\\\\\\\|org#g") +metrics=$(echo $metrics | sed "s#\\\\|org#org#" | sed "s# ##g" | sed "s#\\\org#org#") +metrics=$(printf "$LEGEND\n" | cut -d: -f2 | grep ^org | sed "s#^org#\\\|org#g") +metrics=$(echo $metrics | sed "s#\\\|org#org#" | sed "s# ##g") + +output=$(sed 's#"data":#\n"data":#g' $inputs/$(basename $0 | sed "s#\.sh##").*.out | sed 's#,"#,\n"#g' | grep "^\"_id\"\|^\"metric_name\"\|^\"dst_hostname\"\|^\"status\"\|^\"service_flavour\"\|^\"timestamp\"" | grep -B 5 -A 5 "$metrics") +#exit 0 +if [ ] ; then + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + + for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do + [ "$sf" == "SRM" ] && continue + [ "$sf" == "XROOTD" ] && continue + [ "$sf" == "WEBDAV" ] && continue + nt=0 + for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + nt=$(for t in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do echo $t ; done | wc -l) + [ $nt -lt 10 ] && { nt=0 ; break ; } ; + done + done + [ $nt -lt 10 ] && { echo "
" Warning one or more CE has number of tests less than 10 "
" : nt=$nt >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html ; } ; + + if [ "x$thesite" == "x$THESITE" ] ; then + if [ $nt -lt 10 ] ; then + error_message="$error_message\nERROR one or more CE has number of tests less than 10 nt=$nt \n" + #else + # error_message="$error_message\nINFO All CEs have at least number of tests: nt=$nt \n" + error_message="" + fi + fi + + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +fi # if [ ] ; then + +i=0 +old_sf= +profile_history_site_link="https://monit-grafana.cern.ch/d/000000619/wlcg-sitemon-historical-profiles?orgId=20&var-vo=cms&var-dst_country=All&var-dst_federation=All&var-dst_tier=2&var-dst_experiment_site=${thesite}&var-service_flavour=All&var-profile=CMS_CRITICAL_FULL&var-dst_hostname=All&var-es_dst_hostname=All&var-recomputation=status" +#for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do +# echo $sf +#done +#exit 0 +for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do + i=$(expr $i + 1) + j=0 + [ "$sf" != "WEBDAV" ] && [ "$sf" != "XROOTD" ] && continue + #printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -m 1 -A 5 $t | grep "\"status\"" | cut -d\" -f4 + maxt=0 ; maxh= + #for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + #echo DEBUG $sf $h + #nt=$(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u | wc -l) + nt=$(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep "\"metric_name\"" | cut -d\" -f4 | sort -u | wc -l) + #for t in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + for t in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + m_name=$(echo $t | sed 's#-/# #g' | awk '{print $1}') + L=$(echo $(printf "$LEGEND\n" | grep $m_name | cut -d: -f1)) + if [ "x$L" == "x" ] ; then + echo "$extra_metric_names" | grep -q $t || extra_metric_names="$extra_metric_names
$t" + fi + done + [ $nt -gt $maxt ] && { maxt=$nt ; maxh=$h ; } ; + done +#done +#echo DEBUG maxt=$maxt maxh=$maxh +#exit 0 + lstring= + #for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + if [ "$h" == "$maxh" ] ; then + #[ $i -gt 1 ] && echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + #for t in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + for t in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + m_name=$(echo $t | sed 's#-/# #g' | awk '{print $1}') + L=$(echo $(printf "$LEGEND\n" | grep $m_name | cut -d: -f1)) + [ "x$L" == "x" ] || { + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + twiki_toc=${L}_$(echo $m_name | sed "s#\.#_#g" | sed "s#-#_#g") + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + lstring="${lstring}|$L" + } ; + done + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + done + #echo DEBUG lstring=$lstring +#done +#exit 0 + #for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + profile_history_host_link="https://monit-grafana.cern.ch/d/000000619/wlcg-sitemon-historical-profiles?orgId=20&var-vo=cms&var-dst_country=All&var-dst_federation=All&var-dst_tier=2&var-dst_experiment_site=${thesite}&var-service_flavour=All&var-profile=CMS_CRITICAL_FULL&var-dst_hostname=${h}&var-es_dst_hostname=${h}&var-recomputation=status" + j=$(expr $j + 1) + if [ ] ; then + if [ $i -eq 1 ] ; then # First Service Flavor + if [ $j -eq 1 ] ; then + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + else + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + else + if [ $j -eq 1 ] ; then + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + else + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + fi + fi # if [ ] ; then + im=0 + for legend in $(echo $lstring | sed "s#|# #g") ; do + im=$(expr $im + 1) + t=$(printf "$LEGEND\n" | grep "${legend}:" | cut -d: -f2) + #status=$(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep -m 1 -A 5 $t | grep "\"status\"" | cut -d\" -f4) + #id=$(echo $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep -m 1 -B 5 $t | grep "\"_id\"" | cut -d\" -f4)) + #timestamp=$(echo $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep -B 5 -A 5 $t | grep "\"timestamp\"" | cut -d\" -f3 | cut -d: -f2 | cut -d, -f1 | head -1)) # | cut -d: -f2 | cut -d, -f1 | head -1)) # | grep -m 1 -B 5 -A 5 $t | cut -d\" -f4)) + status=$(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -m 1 -A 5 $t | grep "\"status\"" | cut -d\" -f4) + echo $sf $h $legend t=$t status=$status + if [ "x$status" != "xOK" ] ; then + error_message="$error_message\n[$sf][$h][$t]" + fi + done # for legend in $(echo $lstring | sed "s#|# #g") ; do + done # for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do +done # for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do +if [ "x$error_message" != "x" ] ; then + #printf "$error_message\n" + echo $thesite NOTOK + exit 1 +fi +exit 0 + +# done +# done +# done +#exit 0 + #id=$(echo $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -m 1 -B 5 $t | grep "\"_id\"" | cut -d\" -f4)) + #timestamp=$(echo $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -B 5 -A 5 $t | grep "\"timestamp\"" | cut -d\" -f3 | cut -d: -f2 | cut -d, -f1 | head -1)) # | cut -d: -f2 | cut -d, -f1 | head -1)) # | grep -m 1 -B 5 -A 5 $t | cut -d\" -f4)) + #YmdHM=$(date -d@$(expr $timestamp / 1000) +%Y%m%d_%H:%M) + + #echo DEBUG sf=$sf h=$h legend=$legend t=$t + + #if [ "x$timestamp" == "x" ] ; then + # YmdHM=SPARSE + #else + # YmdHM=$(date -d@$(expr $timestamp / 1000) +%m%d%H%M) + #fi + bgcolor=red + [ "x$status" == "xOK" ] && bgcolor="#90EE90" # green + [ "x$status" == "xCRITICAL" ] && bgcolor=red + [ "x$status" == "xWARNING" ] && bgcolor=orange + [ "x$status" == "xUNKNOWN" ] && bgcolor=grey + [ "x$status" == "x" ] && bgcolor=white + m_name=$(echo $t | sed 's#-/# #g' | awk '{print $1}') + L=$(echo $(printf "$LEGEND\n" | grep $m_name | cut -d: -f1)) + #if [ "x$L" != "x" ] ; then + if [ "x$status" == "x" ] ; then + : + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + #status=NT + #detail_link=SPARSE + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + else + if [ "x$timestamp" == "x" ] ; then + status=SPARSE + detail_link=SPARSE + #else + #sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@id@@#$id#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_id_json}.in > $inputs/${sort_search_id_json}_${now_is} + #eval $(curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_id_json}_${now_is}" 2>/dev/null | python -m json.tool | grep "\"metric_name\"\|\"timestamp\"" | sed "s#:#=#g" | sed "s# ##g" | sed 's#"##g' | sed "s#,##g" | head -2 | sed "s#timestamp=#thetimestamp=#g" | sed "s#/#__slash__#g" | sed "s#=#__equals__#g" | sed "s#metric_name__equals__#metric_name=#" | sed "s#thetimestamp__equals__#thetimestamp=#") + + #detail_link="https://monit-grafana.cern.ch/d/siYq3DxZz/wlcg-sitemon-test-details?orgId=20&var-metric=${metric_name}&var-dst_hostname=${h}&var-timestamp=${thetimestamp}" + #detail_link=$(echo ${detail_link} | sed "s#__slash__#/#g" | sed "s#__equals__#=#g") # / is %2F = is %3D + fi + #echo "" + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + if [ "x$status" != "xOK" ] ; then + #error_message="$error_message\n[$sf][$h][$t] = $status ${detail_link_prefix}${id}\n" + #error_message="$error_message\n[$sf][$h][$t] = $status ${webroot}/$(basename $0)?test_id=$id&site=$thesite\n${webroot}/$(basename $0)\n" + #if [ "x$thesite" == "x$THESITE" ] ; then + error_message="$error_message\n[$sf][$h][$t] = $status \n${detail_link}\n${webroot}/$(basename $0)?test_id=$id&site=$thesite\n\n_id=$id metric_name=${metric_name} thetimestamp=$thetimestamp\n${detail_link}\n${webroot}/$(basename $0)?test_id=$id&site=$thesite\n${webroot}/$(basename $0)\n\nContent of input json\n$([ -f $inputs/${sort_search_id_json}_${now_is} ] && cat $inputs/${sort_search_id_json}_${now_is})" + #fi + #error_message_html="$error_message_html
[$thesite][$sf][$h][$t] = $status
${detail_link}
" + fi + #rm -f $inputs/${sort_search_id_json}_${now_is} + done + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + done +done +if [ "x$error_message" != "x" ] ; then + printf "$error_message\n" +fi +exit 0 +#echo "
SitenameFlavor Host
    $L $L
$thesite $sf $h
   $h
 $sf $h
   $h $L NT
$YmdHM 		$status
T:$YmdHM 		$status
$YmdHM
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +#echo "
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +#done # for thesite in $sites + +echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +for l in $LEGEND ; do + L=$(echo $l | cut -d: -f1) + t=$(echo $l | cut -d: -f2) + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + +done +echo "
LegendMetric Name
$L $t
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "Test numbers > 17 are non-critical CMS tests" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo " Extra Metric Names
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo " $extra_metric_names " >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + +#echo error_message=$error_message +if [ "x$error_message" != "x" ] ; then + #printf "$error_message\n" + #printf "$(/bin/hostname -s) $(basename $0) SAM3 Test Alert\n$error_message\n" | mail -s "ERROR SAM3 Test Alert" $(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + State=ERROR + nline_error_message=$(printf "$error_message\n" | grep "$THESITE\|All CEs have at least number of tests" | wc -l) + [ $nline_error_message -lt 3 ] && State=INFO +#echo DEBUG sending an email + +( + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Subject: State=$State SAM3 Test Alert" + echo "Content-Type: text/html" + echo " Start of Content
Number of Error Message Lines: $nline_error_message
" + printf "$error_message\n" | head -2 + sed -i 's###' $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + echo "" + echo $(/bin/hostname -s) $(basename $0) SAM3 Test Alert "
" + echo ERROR MESSAGE $error_message + echo "$error_message_html" + echo "
" + cat $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + echo +) | /usr/sbin/sendmail -t +else +#echo DEBUG sending an email +#if [ $(expr $(date +%H) % $EVERY_X_HOUR) -eq 0 -a $(date +%M) -lt 15 ] ; then +( + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Subject: OK SAM3 Tests" + echo "Content-Type: text/html" + echo + + cat $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + echo +) # | /usr/sbin/sendmail -t +#fi +fi +rm -f $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +rm -f $inputs/${sitemon_agg_metric_json}_${now_is} +rm -f $inputs/${sort_search_json}_${now_is} +rm -f $inputs/${sort_search_after_json}_${now_is} +rm -f $inputs/$(basename $0 | sed "s#\.sh##").*.out_${now_is} + +exit 0 +####### BEGIN sort_search_json +{"search_type": "query_then_fetch", "index": ["monit_prod_sam3_enr_metric*"], "ignore_unavailable": true} +{"query": {"bool": {"filter": [{"range": {"metadata.timestamp": {"gte": "now-4h/h", "lte": "now", "format": "epoch_millis"}}}, {"query_string": {"analyze_wildcard": true, "query": "data.dst_experiment_site:@@thesite@@"}}]}}, "from": 0, "size": @@size@@, "sort": [{"@@unique_f@@": "desc"},{"@@tie_breakter_id@@": "asc"}]} +####### ENDIN sort_search_json +####### BEGIN sort_search_after_json +search_type": "query_then_fetch", "index": ["monit_prod_sam3_enr_metric*"], "ignore_unavailable": true} +{"query": {"bool": {"filter": [{"range": {"metadata.timestamp": {"gte": "@@gte@@", "lte": "@@lte@@", "format": "epoch_millis"}}}, {"query_string": {"analyze_wildcard": true, "query": "data.dst_experiment_site:@@thesite@@"}}]}}, "from": 0, "size": @@size@@, "search_after": [@@search_after@@], "sort": [{"@@unique_f@@": "desc"},{"@@tie_breakter_id@@": "asc"}]} +####### ENDIN sort_search_after_json +####### BEGIN token_txt +eyJrIjoieEZVYUtzcGkyVnVSaUlUOTRnd3RKWEtPdFRYUDdXQnQiLCJuIjoiY21zLWJvb2Nram9vMyIsImlkIjoxMX0= +####### ENDIN token_txt +####### BEGIN Functions 12345 +# Functions +function readme_org__dot__cms__dot__WN__dash__analysis () { + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-analysis.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__basic () { + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-basic + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__env () { + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-env + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__isolation () { #10 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-isolation + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__squid () { #1 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "[ -f \$SAME_SENSOR_HOME/tests/CE-cms-squid ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-squid \$SAME_SENSOR_HOME/tests" + echo "[ -f \$SAME_SENSOR_HOME/tests/test_frontier.py ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/test_frontier.py \$SAME_SENSOR_HOME/tests" + echo \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-squid.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__frontier () { #2 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "[ -f \$SAME_SENSOR_HOME/tests/CE-cms-frontier ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-frontier \$SAME_SENSOR_HOME/tests" + echo "[ -f \$SAME_SENSOR_HOME/tests/CMSSW_frontier.sh ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/CMSSW_frontier.sh \$SAME_SENSOR_HOME/tests" + echo \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-frontier.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__xrootd__dash__access () { #5 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-xrootd-access.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__xrootd__dash__fallback () { #7 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-xrootd-fallback.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__mc () { #8 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "[ -f \$SAME_SENSOR_HOME/tests/CE-cms-mc ] || cp -pR \$SAME_SENSOR_HOME/../MonteCarlo/tests/CE-cms-mc \$SAME_SENSOR_HOME/tests" + echo "[ -d \$SAME_SENSOR_HOME/cms-MC-test ] || cp -pR \$SAME_SENSOR_HOME/../MonteCarlo/cms-MC-test \$SAME_SENSOR_HOME/" + echo \$SAME_SENSOR_HOME/../MonteCarlo/tests/CE-cms-mc.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__sam__dot__CONDOR__dash__JobSubmit () { #12 + echo "CASE: htcondor" + echo "[0] Go to a condor-g submission machine" + echo "[1] Creat a job submission file, e.g., nagrun.sub with the following content" + cat $inputs/nagrun.sub + echo + echo "[2] Prepare the executable and inputs" + echo "cd" + echo "mkdir JobSubmit/samjob" + echo "cd JobSubmit/samjob" + echo "for f in nagrun.sub nagrun.sh gridjob.tgz wnlogs.tgz ; do" + echo " gfal-copy -r -p gsiftp://cmsio.rc.ufl.edu/cmsuf/t2/operations/samjob/$f ./" + echo "done" + echo + echo cd .. + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo "condor_submit nagrun.sub" + echo "watch condor_q" + echo "echo Check samjob/gridjob.{out,err,log} and wnlogs.tgz" + echo "echo The content of samjob/gridjob.out should be same as the output from the actual SAM CONDOR-JobSubmit test" + + +} + +function readme_org__dot__cms__dot__SRM__dash__GetPFNFromTFC () { #4 + echo "[0] Go to a SAM machine with a grid UI" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "echo Download gridmon" + echo cd \$SAME_SENSOR_HOME/../SRMv2/tests + echo "[ -d python-GridMon ] || { echo Download gridmon ; git clone https://github.com/ARGOeu/python-GridMon.git ; } ;" + echo "[ -d gridmon ] || ln -s python-GridMon/gridmon" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests # to import simplejson and gridmon" + echo "[ -w /var/lib/gridprobes ] || sudo /bin/bash -c \"[ -d /var/lib/gridprobes ] || mkdir /var/lib/gridprobes ; chown \$(id -un):\$(id -gn) /var/lib/gridprobes\"" + + echo \$SAME_SENSOR_HOME/../SE/srmvometrics.py -H cmsio.rc.ufl.edu -m org.cms.SRM-GetPFNFromTFC + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SRM__dash__VOPut () { #11 + echo "[0] Go to a SAM machine with a grid UI" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "echo Download gridmon" + echo cd \$SAME_SENSOR_HOME/../SRMv2/tests + echo "[ -d python-GridMon ] || { echo Download gridmon ; git clone https://github.com/ARGOeu/python-GridMon.git ; } ;" + echo "[ -d gridmon ] || ln -s python-GridMon/gridmon" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests # to import simplejson and gridmon" + echo "[ -w /var/lib/gridprobes ] || sudo /bin/bash -c \"[ -d /var/lib/gridprobes ] || mkdir /var/lib/gridprobes ; chown \$(id -un):\$(id -gn) /var/lib/gridprobes ; [ -f /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ] || mkdir -p /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ; touch /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu/testFileIn.txt\"" + + echo \$SAME_SENSOR_HOME/../SE/srmvometrics.py -H cmsio.rc.ufl.edu -m org.cms.SRM-VOPut + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SRM__dash__VOGet () { #15 + echo "[0] Go to a SAM machine with a grid UI" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "echo Download gridmon" + echo cd \$SAME_SENSOR_HOME/../SRMv2/tests + echo "[ -d python-GridMon ] || { echo Download gridmon ; git clone https://github.com/ARGOeu/python-GridMon.git ; } ;" + echo "[ -d gridmon ] || ln -s python-GridMon/gridmon" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests # to import simplejson and gridmon" + echo "[ -w /var/lib/gridprobes ] || sudo /bin/bash -c \"[ -d /var/lib/gridprobes ] || mkdir /var/lib/gridprobes ; chown \$(id -un):\$(id -gn) /var/lib/gridprobes ; [ -f /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ] || mkdir -p /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ; touch /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu/testFileIn.txt\"" + + echo \$SAME_SENSOR_HOME/../SE/srmvometrics.py -H cmsio.rc.ufl.edu -m org.cms.SRM-VOGet + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__connection () { #6 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__contain () { #14 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__read () { #17 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__version () { #16 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function print_test_output () { + id=$1 + thesite=$2 + now_is=$(date +%s)_${thesite} + sed -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" -e "s|@@id@@|$id|" $inputs/${search_details}.in > ${search_details}_${now_is} + echo "" + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${search_details}_${now_is}" 2>/dev/null | sed 's#"data":#\n"data":#g' | sed 's#,"#,\n"#g' | grep ^\"details\" | cut -d\" -f4- | sed 's|",||' + echo "" + rm -f ${search_details}_${now_is} +} + +####### ENDIN Functions 12345 From d8ea126c347e72edf465787923621e97d2b1400e Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 12:30:46 -0500 Subject: [PATCH 114/192] added the sam3 check to probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 24468d8..40de4b8 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -127,7 +127,14 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ -f $THEPATH/subscribed_sites_$nprod_exp.txt ] ; then thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2) fi - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$thediff\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + sam3result= + for thesite in $thediff ; do + $THEPATH/cms_sam3_check.sh $thesite > $THEPATH/out/cms_sam3_check.${thesite}.txt 2>&1 + status=$? + result=SAM3OK ; [ $status -eq 0 ] || result=SAM3FAIL + sam3result="$sam3result\n$thesite($result)\n" + done + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #exit 1 fi echo "Sites subscribed to the Production Federation: " $nprod From 900f8c1befc88e2173e7b152da15b75dad463e08 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 13:00:54 -0500 Subject: [PATCH 115/192] fixed the json file path --- AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh index 5607e26..d33b444 100755 --- a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh +++ b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh @@ -123,9 +123,9 @@ sort_search_json=cms_sam3_check_sort_search.json sort_search_after_json=cms_sam3_check_sort_search_after.json search_details=es.q.match_sitemon_raw_metric_search_details.json sort_search_id_json=es.q.match_sitemon_raw_metric_sort_search_id.json -perl -n -e 'print if /^####### BEGIN token_txt/ .. /^####### ENDIN token_txt/' < $0 | grep -v "token_txt" > $token_txt -perl -n -e 'print if /^####### BEGIN sort_search_json/ .. /^####### ENDIN sort_search_json/' < $0 | grep -v "sort_search_json" > ${sort_search_json}.in -perl -n -e 'print if /^####### BEGIN sort_search_after_json/ .. /^####### ENDIN sort_search_after_json/' < $0 | grep -v "sort_search_after_json" > ${sort_search_after_json}.in +perl -n -e 'print if /^####### BEGIN token_txt/ .. /^####### ENDIN token_txt/' < $0 | grep -v "token_txt" > $inputs/$token_txt +perl -n -e 'print if /^####### BEGIN sort_search_json/ .. /^####### ENDIN sort_search_json/' < $0 | grep -v "sort_search_json" > $inputs/${sort_search_json}.in +perl -n -e 'print if /^####### BEGIN sort_search_after_json/ .. /^####### ENDIN sort_search_after_json/' < $0 | grep -v "sort_search_after_json" > $inputs/${sort_search_after_json}.in GTE=$(echo $gte | sed 's#/# #' | awk '{print $1}') LTE=$(echo $lte | sed 's#/# #' | awk '{print $1}') @@ -168,11 +168,11 @@ while : ; do i=$(expr $i + 1) if [ $i -eq 1 ] ; then sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_json}.in > $inputs/${sort_search_json} - curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/$token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out rm -f $inputs/${sort_search_json} else sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s|@@search_after@@|$search_after|" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_after_json}.in > $inputs/${sort_search_after_json} - curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_after_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/$token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_after_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out rm -f $inputs/${sort_search_after_json} fi #echo DEBUG $inputs/$(basename $0 | sed "s#\.sh##").$i.out_${now_is} From 540b73c388176c25e6ca44de811b2f1ffd58f807 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 13:07:32 -0500 Subject: [PATCH 116/192] fix a typo --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 40de4b8..fdede4c 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -134,7 +134,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then result=SAM3OK ; [ $status -eq 0 ] || result=SAM3FAIL sam3result="$sam3result\n$thesite($result)\n" done - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #exit 1 fi echo "Sites subscribed to the Production Federation: " $nprod From be3d427b3790f9ad44dc16a31714364fa4642177 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 13:09:23 -0500 Subject: [PATCH 117/192] clean up --- AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh index d33b444..46366dc 100755 --- a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh +++ b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh @@ -371,7 +371,7 @@ for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" done # for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do if [ "x$error_message" != "x" ] ; then #printf "$error_message\n" - echo $thesite NOTOK + #echo $thesite NOTOK exit 1 fi exit 0 From 348564307ec03596c41f6a3a164399ad868cf6e4 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 13:13:48 -0500 Subject: [PATCH 118/192] fix a typo --- AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh index 46366dc..386f71c 100755 --- a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh +++ b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh @@ -510,7 +510,7 @@ search_type": "query_then_fetch", "index": ["monit_prod_sam3_enr_metric*"], "ign {"query": {"bool": {"filter": [{"range": {"metadata.timestamp": {"gte": "@@gte@@", "lte": "@@lte@@", "format": "epoch_millis"}}}, {"query_string": {"analyze_wildcard": true, "query": "data.dst_experiment_site:@@thesite@@"}}]}}, "from": 0, "size": @@size@@, "search_after": [@@search_after@@], "sort": [{"@@unique_f@@": "desc"},{"@@tie_breakter_id@@": "asc"}]} ####### ENDIN sort_search_after_json ####### BEGIN token_txt -eyJrIjoieEZVYUtzcGkyVnVSaUlUOTRnd3RKWEtPdFRYUDdXQnQiLCJuIjoiY21zLWJvb2Nram9vMyIsImlkIjoxMX0= + ####### ENDIN token_txt ####### BEGIN Functions 12345 # Functions From de0209b689aa343548158f1e607670f360aa16bc Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 13:15:12 -0500 Subject: [PATCH 119/192] fix a typo --- AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh index 386f71c..c5be3a1 100755 --- a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh +++ b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh @@ -123,7 +123,7 @@ sort_search_json=cms_sam3_check_sort_search.json sort_search_after_json=cms_sam3_check_sort_search_after.json search_details=es.q.match_sitemon_raw_metric_search_details.json sort_search_id_json=es.q.match_sitemon_raw_metric_sort_search_id.json -perl -n -e 'print if /^####### BEGIN token_txt/ .. /^####### ENDIN token_txt/' < $0 | grep -v "token_txt" > $inputs/$token_txt +[ -f $inputs/$token_txt ] || perl -n -e 'print if /^####### BEGIN token_txt/ .. /^####### ENDIN token_txt/' < $0 | grep -v "token_txt" > $inputs/$token_txt perl -n -e 'print if /^####### BEGIN sort_search_json/ .. /^####### ENDIN sort_search_json/' < $0 | grep -v "sort_search_json" > $inputs/${sort_search_json}.in perl -n -e 'print if /^####### BEGIN sort_search_after_json/ .. /^####### ENDIN sort_search_after_json/' < $0 | grep -v "sort_search_after_json" > $inputs/${sort_search_after_json}.in From 1aadc18cb4b8742d8a3e96b27b89bb681a81a28a Mon Sep 17 00:00:00 2001 From: bockjoo Date: Wed, 9 Nov 2022 14:24:16 -0500 Subject: [PATCH 120/192] site status info added --- AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh | 16 ++++++++++++++++ .../probe_create_send_aaa_metrics.sh | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh index c5be3a1..4b882cc 100755 --- a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh +++ b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh @@ -45,6 +45,7 @@ theprofile=CMS_CRITICAL_FULL #theprofile=CMS_CRITICAL #theprofile=CMS_FULL DBID=9673 +DBID_CMSSST=9475 EVERY_X_HOUR=1 #unique_f=data.metric_name # data.CRAB_JobLogURL ##tie_breaker_id=metadata.kafka_timestamp # data.CRAB_TaskCreationDate @@ -123,9 +124,14 @@ sort_search_json=cms_sam3_check_sort_search.json sort_search_after_json=cms_sam3_check_sort_search_after.json search_details=es.q.match_sitemon_raw_metric_search_details.json sort_search_id_json=es.q.match_sitemon_raw_metric_sort_search_id.json +monit_prod_cmssst_search_json=monit_prod_cmssst_search.json + [ -f $inputs/$token_txt ] || perl -n -e 'print if /^####### BEGIN token_txt/ .. /^####### ENDIN token_txt/' < $0 | grep -v "token_txt" > $inputs/$token_txt perl -n -e 'print if /^####### BEGIN sort_search_json/ .. /^####### ENDIN sort_search_json/' < $0 | grep -v "sort_search_json" > $inputs/${sort_search_json}.in perl -n -e 'print if /^####### BEGIN sort_search_after_json/ .. /^####### ENDIN sort_search_after_json/' < $0 | grep -v "sort_search_after_json" > $inputs/${sort_search_after_json}.in +perl -n -e 'print if /^####### BEGIN monit_prod_cmssst_search_json/ .. /^####### ENDIN monit_prod_cmssst_search_json/' < $0 | grep -v "monit_prod_cmssst_search_json" > $inputs/${monit_prod_cmssst_search_json}.in +startTIS=$(expr $(expr $(date +%s ) / 86400 - 5 ) \* 86400) +limitTIS=$(expr $(expr $(date +%s) / 900 ) \* 900 + 450) GTE=$(echo $gte | sed 's#/# #' | awk '{print $1}') LTE=$(echo $lte | sed 's#/# #' | awk '{print $1}') @@ -141,6 +147,11 @@ if [ $# -lt 1 ] ; then fi [ $# -gt 0 ] && thesite="$1" + +sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" -e "s#@@startTIS@@#$startTIS#" -e "s#@@limitTIS@@#$limitTIS#" $inputs/${monit_prod_cmssst_search_json}.in > $inputs/${monit_prod_cmssst_search_json} +curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/$token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID_CMSSST}/_msearch --data-binary "@$inputs/${monit_prod_cmssst_search_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##")_monit_prod_cmssst_search.out +#echo INFO check $inputs/$(basename $0 | sed "s#\.sh##")_monit_prod_cmssst_search.out +#exit 0 error_message="" #error_message_html="" @@ -501,6 +512,11 @@ rm -f $inputs/${sort_search_after_json}_${now_is} rm -f $inputs/$(basename $0 | sed "s#\.sh##").*.out_${now_is} exit 0 +####### BEGIN monit_prod_cmssst_search_json +{"search_type":"query_then_fetch","index":["monit_prod_cmssst_*"]} +{"query":{"bool":{"must":[{"match_phrase":{"metadata.type":"ssbmetric"}},{"match_phrase":{"metadata.type_prefix":"raw"}},{"match_phrase":{"metadata.path":"sts15min"}}],"filter":{"range":{"metadata.timestamp":{"gte":@@startTIS@@,"lt":@@limitTIS@@,"format":"epoch_second"}}}}},"_source":{"includes":["metadata.timestamp","metadata.kafka_timestamp","data.name","data.status"]},"size":4096,"sort":[{"metadata.timestamp":"desc"},{"metadata.kafka_timestamp":"desc"}]} +####### ENDIN monit_prod_cmssst_search_json + ####### BEGIN sort_search_json {"search_type": "query_then_fetch", "index": ["monit_prod_sam3_enr_metric*"], "ignore_unavailable": true} {"query": {"bool": {"filter": [{"range": {"metadata.timestamp": {"gte": "now-4h/h", "lte": "now", "format": "epoch_millis"}}}, {"query_string": {"analyze_wildcard": true, "query": "data.dst_experiment_site:@@thesite@@"}}]}}, "from": 0, "size": @@size@@, "sort": [{"@@unique_f@@": "desc"},{"@@tie_breakter_id@@": "asc"}]} diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index fdede4c..e859d43 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -131,8 +131,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then for thesite in $thediff ; do $THEPATH/cms_sam3_check.sh $thesite > $THEPATH/out/cms_sam3_check.${thesite}.txt 2>&1 status=$? - result=SAM3OK ; [ $status -eq 0 ] || result=SAM3FAIL - sam3result="$sam3result\n$thesite($result)\n" + result="SAM3 OK" ; [ $status -eq 0 ] || result="SAM3 FAIL" + site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) + sam3result="$sam3result\n$thesite($site_status $result)\n" done printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #exit 1 From e399b44ea2b38b7d9b3423f069838d012653f323 Mon Sep 17 00:00:00 2001 From: bockjoo Date: Tue, 20 Dec 2022 19:00:21 -0500 Subject: [PATCH 121/192] probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index e859d43..388e98c 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -110,7 +110,8 @@ xrdmapc_error=$(grep \\[ $THEPATH/out/xrdmapc_prod_1.txt | grep -v "invalid addr xrdmapc_port0_error=$(grep ":0$\|:0 " $THEPATH/out/xrdmapc_prod_1.txt | awk '{if ($2 == "Man") print $3 ; else print $2}' | sed 's#:0$##g' | while read h ; do grep -v $h:0 $THEPATH/out/xrdmapc_prod_1.txt | grep -q $h $THEPATH/out/xrdmapc_prod_1.txt ; [ $? -eq 0 ] || echo ${h}:0 ; done) if [ $status -eq 0 ] ; then if [ $(date +%M ) -lt 15 ] ; then - printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + : + #printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log fi else printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log @@ -135,7 +136,11 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) sam3result="$sam3result\n$thesite($site_status $result)\n" done - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + if [ "x$thediff" == "xT2_UA_KIPT" ] ; then + echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + else + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + fi #exit 1 fi echo "Sites subscribed to the Production Federation: " $nprod From b9259a3ee8be1f020d614721f62662ce0eafa2cb Mon Sep 17 00:00:00 2001 From: bockjoo Date: Thu, 22 Dec 2022 08:24:01 -0500 Subject: [PATCH 122/192] probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 388e98c..ee3c3d0 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -136,11 +136,12 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) sam3result="$sam3result\n$thesite($site_status $result)\n" done - if [ "x$thediff" == "xT2_UA_KIPT" ] ; then - echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom - else - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom - fi + #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then + echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + #else + # echo $sam3result | grep -q "SAM3 OK" && \ + # printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + #fi #exit 1 fi echo "Sites subscribed to the Production Federation: " $nprod From a6d659e50aaab0c006b1d133fd78aa2c5bbcd669 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 28 Sep 2023 15:40:49 -0400 Subject: [PATCH 123/192] Update create_fedmaps.py skip if site not in vomsfeed --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index 8749841..1c188c5 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -811,6 +811,7 @@ def getXrootdServers () : if 'localhost' in host : continue # give up (site,domain) = findSitename(host) updateHostSitename ( host, site ) + if site not in xrdServers : continue # skip if site not in vomsfeed #print ( " host and site ",host, site ) #print ( "DEBUG xrdServers[site]['endpoints'] ", xrdServers[site]['endpoints'] ) #if '[::ffff:144.16.111.9]:11001' in endpoint : From 1a9e40f5fc4bb732c154c04bf6db53e3d7820e92 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 28 Sep 2023 21:44:10 +0200 Subject: [PATCH 124/192] create_fedmaps.py is updated --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index 8749841..a8936ff 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -42,8 +42,8 @@ # output : {"prod" : [...], "trans" : [...], "nowhere" : [...]} THEPATH='/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' -#VOFEED='http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii' -VOFEED='https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml' +VOFEED='http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii' +#VOFEED='https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml' LOCKFILE=THEPATH+'create_fedmaps.lock' timeout_sec = 10 * 60 @@ -782,7 +782,7 @@ def getXrootdServers () : #print ("DEBUG sites ",len ( sites )) xrdServers = {} for site in sites : - #print ( ' site ',site ) + # print ( ' site ',site ) xrdServers[site] = {} xrdServers[site]['endpoints'] = [] xrdServers[site]['hosts'] = [] @@ -794,7 +794,7 @@ def getXrootdServers () : #print("Line{}: {}".format(count, line.split())) if type (line) is not str : line = line.decode() entries=line.split() - #print ( entries ) + print ( entries ) #if entries[0] == 'Srv' : idx = 0 if entries[1] == 'Man' : idx = 2 @@ -811,7 +811,9 @@ def getXrootdServers () : if 'localhost' in host : continue # give up (site,domain) = findSitename(host) updateHostSitename ( host, site ) - #print ( " host and site ",host, site ) + if site not in xrdServers : continue + #print ( " host and site ",host, site ) + #print ( " xrdServers.keys() ", xrdServers.keys() ) #print ( "DEBUG xrdServers[site]['endpoints'] ", xrdServers[site]['endpoints'] ) #if '[::ffff:144.16.111.9]:11001' in endpoint : # site = 'T2_IN_TIFR' From 579057eaf66898c205652d450913048c465e8622 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 28 Sep 2023 15:50:32 -0400 Subject: [PATCH 125/192] Update README for the merge conflict instruction --- AAAOps/FedProbeSendAAAMetrics/README | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/README b/AAAOps/FedProbeSendAAAMetrics/README index 9668ab1..61121a3 100644 --- a/AAAOps/FedProbeSendAAAMetrics/README +++ b/AAAOps/FedProbeSendAAAMetrics/README @@ -56,3 +56,4 @@ git push origin master On vocms036.cern.ch as root cd /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/ git pull origin master +# if merge conflicts, git checkout --theirs . From d7b9f623fcac918581c38fa973162e048de9b127 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 10 Oct 2023 19:18:44 +0200 Subject: [PATCH 126/192] fix an initial bug --- .../FedProbeSendAAAMetrics/siteLifeStatus.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py index 4784cfb..90da8bd 100644 --- a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py @@ -1,5 +1,10 @@ -import requests, json, os -def siteLifeStatus (site='T2_US_Florida', dbid=9475, gte="now-2h/h",lt="now") : +#!/usr/bin/env python3 +import sys +import os +import requests +import json + +def siteLifeStatus (site='T2_US_Florida', dbid=9475, gte="now-6h/h",lt="now") : ''' gets a site life status during last 2 hours ''' @@ -67,11 +72,22 @@ def siteLifeStatus (site='T2_US_Florida', dbid=9475, gte="now-2h/h",lt="now") : #print ( 'Number of Responses ', len ( result["responses"] ) ) - response = result["responses"] - #print ( 'Total hits ',r['hits']['total']['value'] ) + response = result["responses"][0] + #print ( 'Total hits ',response['hits']['total']['value'] ) siteStatus={} - for hit in r['hits']['hits'] : + #print ( type ( response['hits']['hits'] ) ) + for hit in response['hits']['hits'] : #print ("Site ",hit['_source']['data']['name'], " Status ",hit['_source']['data']['status']) siteStatus[hit['_source']['data']['name']] = hit['_source']['data']['status'] return siteStatus[site] +if __name__ == "__main__": + if len(sys.argv) > 1 : + site = sys.argv[1] + status = siteLifeStatus(site=site) + print (status) + sys.exit(0) + status = siteLifeStatus() + print (status) + + From 1be13c29ad9808a53ef0a8a084fb155873ca86ea Mon Sep 17 00:00:00 2001 From: root Date: Tue, 10 Oct 2023 19:28:46 +0200 Subject: [PATCH 127/192] update the cron to include the site life status --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index ee3c3d0..454e4e8 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -134,7 +134,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then status=$? result="SAM3 OK" ; [ $status -eq 0 ] || result="SAM3 FAIL" site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) - sam3result="$sam3result\n$thesite($site_status $result)\n" + siteLifeStatus=$(export GRAFANA_VIEWER_TOKEN=$(cat $THEPATH/token.txt) ; python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $thesite) + sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" + done #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From d73edf04d9b07bb2da939c2d4b1b8c9927764ce6 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 11 Oct 2023 02:19:11 +0200 Subject: [PATCH 128/192] test update --- AAAOps/FedProbeSendAAAMetrics/listSrv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/listSrv.sh b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh index 923006a..3a1f11e 100755 --- a/AAAOps/FedProbeSendAAAMetrics/listSrv.sh +++ b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh @@ -1,7 +1,7 @@ #!/bin/bash ipv= ; ipos=2 [ $# -gt 0 ] && { ipv=6 ; ipos=3 ; } ; - +[ -f out/xrdmapc_all_0.txt ] || exit 1 for h in $( grep Srv out/xrdmapc_all_0.txt | awk '{print $2}' | cut -d: -f1 | grep -v \\[ | sort -u ) ; do echo $h $(ping${ipv} -c 1 $h | grep ^PING | cut -d\( -f${ipos} | cut -d\) -f1) From 603eafec6b324b225d46a102812b0c69f2740f39 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Tue, 19 Dec 2023 15:29:02 -0500 Subject: [PATCH 129/192] Update probe_create_send_aaa_metrics.sh To add noreply@cern.ch --- .../probe_create_send_aaa_metrics.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 454e4e8..43cafc1 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -8,10 +8,10 @@ GRAFANA_PAGE="https://monit-grafana.cern.ch/d/5njhdTrWk/site-subscription?from=n export X509_USER_PROXY=$HOME/.globus/slsprobe.proxy if [ -f $X509_USER_PROXY ] ; then if [ $(voms-proxy-info -timeleft 2>/dev/null) -lt 3600 ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\n$(voms-proxy-info -all | sed 's#%#%%#g')" | mail -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 1" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\n$(voms-proxy-info -all | sed 's#%#%%#g')" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 1" $notifytowhom fi else - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\nIt does not exist" | mail -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 2" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\nIt does not exist" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 2" $notifytowhom fi notifytowhom=bockjoo__AT__gmail__dot__com export PYTHONPATH=$PYTHONPATH:$THEPATH/CMSMonitoring/src/python/ @@ -21,17 +21,17 @@ notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') python3 $THEPATH/create_fedmaps.py > $THEPATH/create_fedmaps.log 2>&1 if [ $? -ne 0 ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 $THEPATH/create_fedmaps.py\n\n$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 $THEPATH/create_fedmaps.py\n\n$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi if [ ! -r $FED_json ]; then echo "We have a problem creating JSON file.\n" - printf "$(/bin/hostname -s) $(basename $0) We have a problem creating JSON file.\n\nn$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem creating JSON file.\n\nn$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom exit 1 fi cat $FED_json | python -m json.tool 2>/dev/null 1>/dev/null if [ $? -ne 0 ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have a problem with $FED_json.\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $FED_json + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $FED_json.\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $FED_json exit 1 fi [ -d $(dirname $THELOG) ] || mkdir -p $(dirname $THELOG) @@ -57,6 +57,7 @@ is_int() { } timeout=10 echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Reply-To: noreply@cern.ch" echo "Subject: XRootD Version Role List" echo "Content-Type: text/html" echo "" @@ -111,7 +112,7 @@ xrdmapc_port0_error=$(grep ":0$\|:0 " $THEPATH/out/xrdmapc_prod_1.txt | awk '{if if [ $status -eq 0 ] ; then if [ $(date +%M ) -lt 15 ] ; then : - #printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + #printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -r noreply@cern.ch -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log fi else printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log @@ -139,10 +140,10 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then done #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then - echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #else # echo $sam3result | grep -q "SAM3 OK" && \ - # printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + # printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #fi #exit 1 fi From d56576e3bc0b30be32045a5c8ea602ce5b4743e6 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Tue, 19 Dec 2023 15:31:59 -0500 Subject: [PATCH 130/192] Update uploadmetricGeneral.sh To add noreply@cern.ch --- .../RedirectorServiceAvailability/uploadmetricGeneral.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh index 5165af4..5a1603b 100755 --- a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -26,7 +26,7 @@ echo INFO executing XRDFED-kibana-probe_JSON_General.py python3 XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 status=$? if [ $status -ne 0 ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 XRDFED-kibana-probe_JSON_General.py\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 XRDFED-kibana-probe_JSON_General.py\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi if [ ] ; then @@ -44,7 +44,7 @@ for r in $rdirs ; do fi done if [ "x$rdirs_degraded" != "x" ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdcp -d 1 -f -DIReadCacheSize 0 -DIRedirCntTimeout 180 -DIConnectTimeout 30 -DITransactionTimeout 60 -DIRequestTimeout 60 root://rdir//store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root /dev/null\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdcp -d 1 -f -DIReadCacheSize 0 -DIRedirCntTimeout 180 -DIConnectTimeout 30 -DITransactionTimeout 60 -DIRequestTimeout 60 root://rdir//store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root /dev/null\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi fi @@ -52,7 +52,7 @@ fi for r in $rdirs ; do grep xrdmapc $logs/XRDFED_probe_json.log | grep "$r" | grep -q "\[ERROR\] Operation expired" if [ $? -eq 0 ] ; then - printf "$(/bin/hostname -s) $(basename $0) We have the Operation expired with $r\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdmapc --list all $r\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + printf "$(/bin/hostname -s) $(basename $0) We have the Operation expired with $r\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdmapc --list all $r\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom fi done echo INFO $logs/XRDFED_probe_json.log @@ -69,7 +69,7 @@ if [ -f $thelog ] ; then a=1 [ $status -eq 0 ] && { grep -q -i "caught overall timeout" $logs/XRDFED_probe_json.log ; [ $(expr $a + $?) -eq $a ] && status=1 ; } ; if [ $status -ne 0 ] ; then - printf "$(/bin/hostname) $(basename $0)\n$(date)\n$(ls -al $thelog )\n$(cat $thelog)\n" | mail -s "$(/bin/hostname) $(basename $thelog)" $(echo $notifytowhom | sed 's#NOSPAMPLEASE##g') + printf "$(/bin/hostname) $(basename $0)\n$(date)\n$(ls -al $thelog )\n$(cat $thelog)\n" | mail -r noreply@cern.ch -s "$(/bin/hostname) $(basename $thelog)" $(echo $notifytowhom | sed 's#NOSPAMPLEASE##g') fi fi From 2f1af611ef12e2d025c5ac04571cb316fc118af1 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sun, 31 Mar 2024 09:00:38 -0400 Subject: [PATCH 131/192] Update aaa_federation.py for RHEL8 default stomp 7.0.0 --- .../FedProbeSendAAAMetrics/aaa_federation.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py index 8c2bc32..4a65b00 100755 --- a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py +++ b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py @@ -2,7 +2,8 @@ import re import sys -__requires__ = 'stomp.py==4.1.21' +#RHEL8 default is 7.0.0 __requires__ = 'stomp.py==4.1.21' +__requires__ = 'stomp.py==7.0.0' from pkg_resources import load_entry_point import stomp @@ -22,6 +23,12 @@ from uuid import uuid4 from urllib.request import urlopen import time +#RHEL8 +# CMSMonitoring modules +try: + from CMSMonitoring.StompAMQ7 import StompAMQ7 +except ImportError: + print("ERROR: Could not import StompAMQ7") print("stomp version", stomp.__version__) @@ -118,6 +125,9 @@ def createFlattenJson(json_file): f.write("%s\n" %my_json) if __name__ == "__main__": + #RHEL8 + ts = int(time.time()) * 1000 + #RHEL8 optmgr = OptionParser() opts = optmgr.parser.parse_args() payload = parseFederationJson() @@ -135,15 +145,22 @@ def createFlattenJson(json_file): producer = creds['producer'] topic = creds['topic'] hosts = [(host, port)] - if creds and StompAMQ: - amq = StompAMQ(username, password, producer, topic, key=None, + #RHEL8 + doc_type = creds['type'] + #RHEL8 if creds and StompAMQ: + #RHEL8 amq = StompAMQ(username, password, producer, topic, key=None, + #RHEL8 cert=None, validation_schema=None, host_and_ports=[(host, port)]) + if creds and StompAMQ7: + amq = StompAMQ7(username, password, producer, topic, key=None, cert=None, validation_schema=None, host_and_ports=[(host, port)]) eod = False wait_seconds = 10 while not eod: messages = [] for d in payload: - notif,_,_ = amq.make_notification(d, "aaa_federations_document", dataSubfield=None) + #RHEL8 notif,_,_ = amq.make_notification(d, "aaa_federations_document", dataSubfield=None) + notif,_,_ = amq.make_notification(payload=d, doc_type=doc_type, + producer=producer, ts=ts) messages.append(notif) if messages: print(messages) From 859e3661a92483f52407ffac33ee36bcdc9a4af2 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Tue, 27 Aug 2024 11:13:46 -0400 Subject: [PATCH 132/192] Update siteLifeStatus.py index_name and DBID update --- AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py index 90da8bd..6a02639 100644 --- a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py @@ -4,13 +4,15 @@ import requests import json -def siteLifeStatus (site='T2_US_Florida', dbid=9475, gte="now-6h/h",lt="now") : +#def siteLifeStatus (site='T2_US_Florida', dbid=9475, gte="now-6h/h",lt="now") : +def siteLifeStatus (site='T2_US_Florida', dbid=9980, gte="now-6h/h",lt="now") : ''' gets a site life status during last 2 hours ''' url = "https://monit-grafana.cern.ch/api/datasources/proxy/"+str(dbid)+"/_msearch" index_name = "monit_prod_cms_raw_aaa-ng*" # XRD Collector Monitoring - index_name = "monit_prod_cmssst_*" # Site Status + index_name = "monit_prod_cmssst_*" # 9475 Site Status + index_name = "monit_prod_cmssst_raw_ssbmetric-*" # 9980 Site Status payload_index_props = { "search_type": "query_then_fetch", "index": [ index_name ], From fe6c602a946377ba40b5e5c881c4d8e501a97e52 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Tue, 27 Aug 2024 11:22:54 -0400 Subject: [PATCH 133/192] Update aaa_federation.py --- AAAOps/FedProbeSendAAAMetrics/aaa_federation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py index 4a65b00..5479e54 100755 --- a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py +++ b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py @@ -29,6 +29,7 @@ from CMSMonitoring.StompAMQ7 import StompAMQ7 except ImportError: print("ERROR: Could not import StompAMQ7") + sys.exit(1) print("stomp version", stomp.__version__) From bb046350b838a55e197ede1c72cf5313e9db358f Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 28 Aug 2024 06:36:01 -0400 Subject: [PATCH 134/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 43cafc1..1d73413 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -129,6 +129,13 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ -f $THEPATH/subscribed_sites_$nprod_exp.txt ] ; then thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2) fi + day_of_week=$(date +%u%H%M%S) + if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1000013 ] ; then + printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt + fi + if [ ! -f $THEPATH/site_aaa_status.txt ] ; then + printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt + fi sam3result= for thesite in $thediff ; do $THEPATH/cms_sam3_check.sh $thesite > $THEPATH/out/cms_sam3_check.${thesite}.txt 2>&1 @@ -137,7 +144,17 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) siteLifeStatus=$(export GRAFANA_VIEWER_TOKEN=$(cat $THEPATH/token.txt) ; python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $thesite) sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" - + expected=Yes + if [ "$result" == "SAM OK" ] ; then + [ $(echo $siteLifeStatus | grep -q "wait\|morgue" ; echo $?) -eq 0 ] && expected=No + fi + if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then + siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) + WkCount=$(expr $(echo $siteline | awk '{print $5}') + 1) + printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt + else + printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt + fi done #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From 3b69581a222930cb21d4ff63cc322e0600dae1c4 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 28 Aug 2024 06:45:15 -0400 Subject: [PATCH 135/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 1d73413..600f153 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -157,7 +157,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then fi done #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then - echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #else # echo $sam3result | grep -q "SAM3 OK" && \ # printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From 453736eb62dac38b21e96b2ac2ca12b6b78a2761 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 28 Aug 2024 07:15:05 -0400 Subject: [PATCH 136/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 600f153..7098024 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -151,6 +151,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) WkCount=$(expr $(echo $siteline | awk '{print $5}') + 1) + sed -i "/$(echo $siteline | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt From 931a8dc00a8197fbc2ff27df629c00804f7ee1cb Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 28 Aug 2024 08:24:43 -0400 Subject: [PATCH 137/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 7098024..8aaf13d 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -150,7 +150,8 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then fi if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) - WkCount=$(expr $(echo $siteline | awk '{print $5}') + 1) + WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') + WkCount=$(expr $WkCount_previous + 1) sed -i "/$(echo $siteline | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt else From 2493039d0718512d1534bc08a879455c744b06d4 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 28 Aug 2024 10:15:34 -0400 Subject: [PATCH 138/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 8aaf13d..e1bfa92 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -146,7 +146,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" expected=Yes if [ "$result" == "SAM OK" ] ; then - [ $(echo $siteLifeStatus | grep -q "wait\|morgue" ; echo $?) -eq 0 ] && expected=No + [ $(echo "$siteLifeStatus" | grep -q "wait\|morgue" ; echo $?) -eq 0 ] && expected=No fi if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) From bcf7bf0b839a4bd68dc7dd1a346746009bd142e0 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 28 Aug 2024 10:16:50 -0400 Subject: [PATCH 139/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index e1bfa92..02c89e1 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -143,6 +143,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then result="SAM3 OK" ; [ $status -eq 0 ] || result="SAM3 FAIL" site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) siteLifeStatus=$(export GRAFANA_VIEWER_TOKEN=$(cat $THEPATH/token.txt) ; python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $thesite) + if [ "x$siteLifeStatus" == "x" ] ; then + siteLifeStatus=UNKNOWN + fi sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" expected=Yes if [ "$result" == "SAM OK" ] ; then From 18dfd29384bdb5f06fda183fc12e64dad96e69af Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 28 Aug 2024 10:36:30 -0400 Subject: [PATCH 140/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 02c89e1..e2ad752 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -155,7 +155,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') WkCount=$(expr $WkCount_previous + 1) - sed -i "/$(echo $siteline | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt + sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt From 0a9e8e73894589ceb260cb78775d17ebb0a5254c Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 05:54:14 -0400 Subject: [PATCH 141/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index e2ad752..672d1bb 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -131,9 +131,15 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then fi day_of_week=$(date +%u%H%M%S) if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1000013 ] ; then + echo "" > $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi if [ ! -f $THEPATH/site_aaa_status.txt ] ; then + echo "" > $THEPATH/site_aaa_status.html + echo "
Site Life Status SAM Status WkCount Expected
" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi sam3result= @@ -157,12 +163,19 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then WkCount=$(expr $WkCount_previous + 1) sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt + sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt + echo "" >> $THEPATH/site_aaa_status.html fi done + if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1000013 ] ; then + echo "
Site Life Status SAM Status WkCount Expected
$thesite $siteLifeStatus $result $WkCount $expected
$thesite $siteLifeStatus $result 1 $expected
" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then - echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #else # echo $sam3result | grep -q "SAM3 OK" && \ # printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From 9fab446762f9de86b1e950b1fec92d804d606f0d Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 06:46:53 -0400 Subject: [PATCH 142/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 672d1bb..b4aae3a 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -130,7 +130,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2) fi day_of_week=$(date +%u%H%M%S) - if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1000013 ] ; then + if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1002800 ] ; then echo "" > $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html From 300da0b149db7af7ce26863d3637938fa6894285 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 07:25:28 -0400 Subject: [PATCH 143/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index b4aae3a..6bf9264 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -170,10 +170,8 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "" >> $THEPATH/site_aaa_status.html fi done - if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1000013 ] ; then - echo "
Site Life Status SAM Status WkCount Expected
$thesite $siteLifeStatus $result 1 $expected
" >> $THEPATH/site_aaa_status.html - echo "" >> $THEPATH/site_aaa_status.html - fi + grep -q "" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html + grep -q "" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #else From 8ee2f6eca9db1578151534e3963206317ed083bc Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 07:28:56 -0400 Subject: [PATCH 144/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 6bf9264..9808a67 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -136,9 +136,11 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo " Site Life Status SAM Status WkCount Expected " >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi - if [ ! -f $THEPATH/site_aaa_status.txt ] ; then + if [ ! -f $THEPATH/site_aaa_status.html ] ; then echo "" > $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html + fi + if [ ! -f $THEPATH/site_aaa_status.txt ] ; then echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi From 0aedb0b1df5affd96790df4569a96a6dbb0ea4de Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 08:05:50 -0400 Subject: [PATCH 145/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 9808a67..7ce1998 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -172,8 +172,10 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "" >> $THEPATH/site_aaa_status.html fi done - grep -q "
Site Life Status SAM Status WkCount Expected
$thesite $siteLifeStatus $result 1 $expected
" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html - grep -q "" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html + if [ -f $THEPATH/site__aaa_status.html ] ; then + grep -q "" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html + grep -q "" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html + fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom #else From 2d294c8b5b7481503b970687b19c9dcda0a1dc4f Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 11:03:07 -0400 Subject: [PATCH 146/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 7ce1998..2c79038 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -178,6 +178,15 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + if [ $(echo $sam3result | grep -q "SAM3 OK" ; echo $?) -eq 0 ] ; then + ( + echo "To: $notifytowhom" + echo "Subject: echo $(basename $0) on $(/bin/hostname -s)" + echo "Reply-to: noreply@cern.ch" + echo "Content-Type: text/html" + cat $THEPATH/site_aaa_status.html + ) | /usr/sbin/sendmail -t + fi #else # echo $sam3result | grep -q "SAM3 OK" && \ # printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From 9ac41bf44414b12dfe4555a6516f451faed1097b Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 11:20:48 -0400 Subject: [PATCH 147/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 2c79038..c3b0bee 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -172,9 +172,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo " $thesite $siteLifeStatus $result 1 $expected " >> $THEPATH/site_aaa_status.html fi done - if [ -f $THEPATH/site__aaa_status.html ] ; then - grep -q "" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html - grep -q "" $THEPATH/site__aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html + if [ -f $THEPATH/site_aaa_status.html ] ; then + grep -q "" $THEPATH/site_aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html + grep -q "" $THEPATH/site_aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From 82904c692536ef0b33242507c50fa831c794efff Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 29 Aug 2024 17:00:55 -0400 Subject: [PATCH 148/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index c3b0bee..9934692 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -173,8 +173,12 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then fi done if [ -f $THEPATH/site_aaa_status.html ] ; then - grep -q "" $THEPATH/site_aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html - grep -q "" $THEPATH/site_aaa_status.html || echo "" >> $THEPATH/site_aaa_status.html + thestring="" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From 836daf8e28140c33d90af540838f7765c9fe1fe3 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 30 Aug 2024 05:09:44 -0400 Subject: [PATCH 149/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 9934692..dd4e6d0 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -157,7 +157,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" expected=Yes if [ "$result" == "SAM OK" ] ; then - [ $(echo "$siteLifeStatus" | grep -q "wait\|morgue" ; echo $?) -eq 0 ] && expected=No + [ $(echo "$siteLifeStatus" | grep -q "enabled" ; echo $?) -eq 0 ] && expected=No fi if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) From 7cb428afd76c01d868490487c5c04f1595a9661d Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 30 Aug 2024 05:50:28 -0400 Subject: [PATCH 150/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index dd4e6d0..cc662e6 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -127,7 +127,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ $nprod_exp -gt $nprod ] ; then thediff= if [ -f $THEPATH/subscribed_sites_$nprod_exp.txt ] ; then - thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2) + thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2 | sort -u) fi day_of_week=$(date +%u%H%M%S) if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1002800 ] ; then From 6268efc897bd67d269adcd5394c9b83d9cf80ac2 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 30 Aug 2024 06:07:26 -0400 Subject: [PATCH 151/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index cc662e6..44fd21b 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -185,7 +185,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ $(echo $sam3result | grep -q "SAM3 OK" ; echo $?) -eq 0 ] ; then ( echo "To: $notifytowhom" - echo "Subject: echo $(basename $0) on $(/bin/hostname -s)" + echo "Subject: $(basename $0) on $(/bin/hostname -s)" echo "Reply-to: noreply@cern.ch" echo "Content-Type: text/html" cat $THEPATH/site_aaa_status.html From bbef7bde73f84a026ef1a4781baa2a4be8d7aed4 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 30 Aug 2024 10:00:16 -0400 Subject: [PATCH 152/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 44fd21b..8c33b8d 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -133,7 +133,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1002800 ] ; then echo "" > $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi if [ ! -f $THEPATH/site_aaa_status.html ] ; then @@ -141,7 +141,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "
Site Life Status SAM Status WkCount Expected
Site Life Status SAM Status WkCount Expected SAM Critical
" >> $THEPATH/site_aaa_status.html fi if [ ! -f $THEPATH/site_aaa_status.txt ] ; then - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi sam3result= @@ -159,6 +159,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ "$result" == "SAM OK" ] ; then [ $(echo "$siteLifeStatus" | grep -q "enabled" ; echo $?) -eq 0 ] && expected=No fi + sam_critical_host_tests=$(grep CRITICAL $THEPATH/out/cms_sam3_check.${thesite}.txt | awk '{print $2" "$4"
"}') if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') @@ -166,10 +167,10 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html fi done if [ -f $THEPATH/site_aaa_status.html ] ; then From 02c371743ac2b956fce9d1b8566303b2311386a2 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 30 Aug 2024 11:11:40 -0400 Subject: [PATCH 153/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 8c33b8d..5c085c3 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -167,6 +167,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt From fab22d06b81728327cabf3c015392021dc3485a3 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 30 Aug 2024 11:38:03 -0400 Subject: [PATCH 154/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 5c085c3..f799c6f 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -167,11 +167,11 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html - echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html - echo "" >> $THEPATH/site_aaa_status.html + #echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html fi done if [ -f $THEPATH/site_aaa_status.html ] ; then From 80711dd69a0f823951dd2b32c5c38220cb9cc886 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Fri, 30 Aug 2024 11:53:11 -0400 Subject: [PATCH 155/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index f799c6f..1460dc6 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -183,7 +183,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "" >> $THEPATH/site_aaa_status.html fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then - echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + #echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom if [ $(echo $sam3result | grep -q "SAM3 OK" ; echo $?) -eq 0 ] ; then ( echo "To: $notifytowhom" From f46503c27bf7c98bffd3a964ccba88715d5c11df Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 31 Aug 2024 12:53:35 -0400 Subject: [PATCH 156/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 1460dc6..9fa2ea5 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -182,6 +182,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "
Site Life Status SAM Status WkCount Expected
Site Life Status SAM Status WkCount Expected SAM Critical
$thesite $siteLifeStatus $result $WkCount $expected
$thesite $siteLifeStatus $result $WkCount $expected $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result 1 $expected
$thesite $siteLifeStatus $result 1 $expected $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result $WkCount $expected $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result $WkCount $expected $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result $WkCount $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result 1 $expected $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result 1 $expected $sam_critical_host_tests
" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html fi + if [ $? -ne 0 ] ; then + sed -i "1s|^| Sites with Frequent Subscription Failure \n|" $THEPATH/site_aaa_status.html + fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then #echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom if [ $(echo $sam3result | grep -q "SAM3 OK" ; echo $?) -eq 0 ] ; then From 2f3cac2720f71142cb5e82f307ffad756b04a657 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 31 Aug 2024 13:21:55 -0400 Subject: [PATCH 157/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 9fa2ea5..1933509 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -183,6 +183,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "" >> $THEPATH/site_aaa_status.html fi if [ $? -ne 0 ] ; then + echo DEBUG updating with "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html sed -i "1s|^| Sites with Frequent Subscription Failure \n|" $THEPATH/site_aaa_status.html fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then From d22cc9bd671052a6c715e3cf617afe5a95dbe34e Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 31 Aug 2024 14:28:15 -0400 Subject: [PATCH 158/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 1933509..23f5394 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -182,6 +182,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html fi + grep -q "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html if [ $? -ne 0 ] ; then echo DEBUG updating with "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html sed -i "1s|^| Sites with Frequent Subscription Failure \n|" $THEPATH/site_aaa_status.html From ebbaae189f9876664f67419e4c154861849bf00c Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Mon, 2 Sep 2024 15:31:57 -0400 Subject: [PATCH 159/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 23f5394..92c86d5 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -123,6 +123,22 @@ nprod_exp=54 if [ -f $THEPATH/check_subscribed_sites.sh ] ; then subscribed_sites=$($THEPATH/check_subscribed_sites.sh) nprod=$(printf "$subscribed_sites\n" | tail -1 | awk '{print $2}') + if [ $nprod -lt 10 ] ; then + echo Warning something is wrong + ( + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Reply-To: noreply@cern.ch" + echo "Subject: XRootD Version Role List" + echo "Content-Type: text/html" + echo "
"
+	 echo $THEPATH/create_fedmaps.log
+	 cat $THEPATH/create_fedmaps.log
+	 echo ; echo ; echo
+	 echo $THEPATH/fed.json
+	 cat $THEPATH/fed.json
+	 echo "
" + ) | /usr/sbin/sendmail -t + fi printf "$subscribed_sites\n" > $THEPATH/subscribed_sites_$nprod.txt if [ $nprod_exp -gt $nprod ] ; then thediff= From 3ab2d90e514f78772442b1937e190e76ac99e832 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 4 Sep 2024 19:04:42 -0400 Subject: [PATCH 160/192] Update siteLifeStatus.py --- AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py index 6a02639..89e646e 100644 --- a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py @@ -5,7 +5,7 @@ import json #def siteLifeStatus (site='T2_US_Florida', dbid=9475, gte="now-6h/h",lt="now") : -def siteLifeStatus (site='T2_US_Florida', dbid=9980, gte="now-6h/h",lt="now") : +def siteLifeStatus (site='T2_US_Florida', what='sts15min', dbid=9980, gte="now-23h/h",lte="now") : ''' gets a site life status during last 2 hours ''' @@ -63,7 +63,9 @@ def siteLifeStatus (site='T2_US_Florida', dbid=9980, gte="now-6h/h",lt="now") : ] } payload_query["query"]["bool"]["filter"]["range"]["metadata.timestamp"]["gte"] = gte - payload_query["query"]["bool"]["filter"]["range"]["metadata.timestamp"]["lt"] = lt + payload_query["query"]["bool"]["filter"]["range"]["metadata.timestamp"]["lt"] = lte + # down15min or sts15min + payload_query["query"]["bool"]["must"][2]["match_phrase"]["metadata.monit_hdfs_path"] = what #print(payload_query) payload = json.dumps(payload_index_props) + " \n" + json.dumps(payload_query) + "\n" headers = { @@ -84,6 +86,12 @@ def siteLifeStatus (site='T2_US_Florida', dbid=9980, gte="now-6h/h",lt="now") : return siteStatus[site] if __name__ == "__main__": + if len(sys.argv) > 2 : + site = sys.argv[1] + what = sys.argv[2] + status = siteLifeStatus(site=site, what=what) + print (status) + sys.exit(0) if len(sys.argv) > 1 : site = sys.argv[1] status = siteLifeStatus(site=site) From b4e7e2800bc3286814c37c79536f0026e4566c8e Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 4 Sep 2024 19:20:45 -0400 Subject: [PATCH 161/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 92c86d5..300ef55 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -146,10 +146,10 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2 | sort -u) fi day_of_week=$(date +%u%H%M%S) - if [ $day_of_week -gt 1000000 -a $day_of_week -lt 1002800 ] ; then + if [ $day_of_week -gt 1014000 -a $day_of_week -lt 1002800 ] ; then echo "" > $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi if [ ! -f $THEPATH/site_aaa_status.html ] ; then @@ -157,7 +157,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "
Site Life Status SAM Status WkCount Expected SAM Critical
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
" >> $THEPATH/site_aaa_status.html fi if [ ! -f $THEPATH/site_aaa_status.txt ] ; then - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi sam3result= @@ -167,9 +167,8 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then result="SAM3 OK" ; [ $status -eq 0 ] || result="SAM3 FAIL" site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) siteLifeStatus=$(export GRAFANA_VIEWER_TOKEN=$(cat $THEPATH/token.txt) ; python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $thesite) - if [ "x$siteLifeStatus" == "x" ] ; then - siteLifeStatus=UNKNOWN - fi + siteDownStatus=$(export GRAFANA_VIEWER_TOKEN=$(cat $THEPATH/token.txt) ; python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $thesite "down15min") + sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" expected=Yes if [ "$result" == "SAM OK" ] ; then @@ -184,10 +183,10 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html #echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt - echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html fi done if [ -f $THEPATH/site_aaa_status.html ] ; then @@ -201,7 +200,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then grep -q "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html if [ $? -ne 0 ] ; then echo DEBUG updating with "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html - sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then #echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom From 117652e24cc6b5875dc17d78a28c1b37381a3a5b Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 4 Sep 2024 19:34:36 -0400 Subject: [PATCH 162/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 300ef55..7b61cbb 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -155,7 +155,18 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then if [ ! -f $THEPATH/site_aaa_status.html ] ; then echo "" > $THEPATH/site_aaa_status.html echo "
Site Life Status SAM Status WkCount Expected SAM Critical
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
$thesite $siteLifeStatus $result $WkCount $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result $siteDownStatus $WkCount $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result 1 $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result $siteDownStatus 1 $expected $sam_critical_host_tests
Sites with Frequent Subscription Failure
Sites with Frequent Subscription Failure
" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html fi + thestring="" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
Site
" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html + sed -i "1s/^/
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
\n/" $THEPATH/site_aaa_status.html + sed -i "1s/^/\n/" $THEPATH/site_aaa_status.html + if [ ! -f $THEPATH/site_aaa_status.txt ] ; then echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt From c413dee85b6636eec4ade9dbe14900a03f1e03cb Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 4 Sep 2024 20:11:27 -0400 Subject: [PATCH 163/192] Update siteLifeStatus.py --- AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py index 89e646e..3b44766 100644 --- a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py @@ -83,6 +83,8 @@ def siteLifeStatus (site='T2_US_Florida', what='sts15min', dbid=9980, gte="now-2 for hit in response['hits']['hits'] : #print ("Site ",hit['_source']['data']['name'], " Status ",hit['_source']['data']['status']) siteStatus[hit['_source']['data']['name']] = hit['_source']['data']['status'] + if len(siteStatus.keys()) == 0 : + return 'UNKNOWN' return siteStatus[site] if __name__ == "__main__": From 9eeeb71a56d4e8c23f60a459fe1b8d66aeb28c3b Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 4 Sep 2024 20:23:21 -0400 Subject: [PATCH 164/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 7b61cbb..452fcfa 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -157,12 +157,17 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html fi + + thestring="Sites with Frequent Subscription Failure" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
Site
" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html sed -i "1s/^/
Sites with Frequent Subscription Failure
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
\n/" $THEPATH/site_aaa_status.html sed -i "1s/^/\n/" $THEPATH/site_aaa_status.html @@ -208,11 +213,11 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "
" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html fi - grep -q "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html - if [ $? -ne 0 ] ; then - echo DEBUG updating with "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html - sed -i "1s|^| Sites with Frequent Subscription Failure \n|" $THEPATH/site_aaa_status.html - fi + #grep -q "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html + #if [ $? -ne 0 ] ; then + # echo DEBUG updating with "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html + # sed -i "1s|^| Sites with Frequent Subscription Failure \n|" $THEPATH/site_aaa_status.html + #fi #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then #echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom if [ $(echo $sam3result | grep -q "SAM3 OK" ; echo $?) -eq 0 ] ; then From 8d0153ae7823faa9eafb02ac7468efa6ed186ba0 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 4 Sep 2024 20:31:27 -0400 Subject: [PATCH 165/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 452fcfa..4a20d5d 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -160,15 +160,15 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then thestring="Sites with Frequent Subscription Failure" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html - thestring="Site" + thestring="Site" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html - sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html - sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html sed -i "1s/^/
Sites with Frequent Subscription Failure
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
Sites with Frequent Subscription Failure
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
\n/" $THEPATH/site_aaa_status.html sed -i "1s/^/\n/" $THEPATH/site_aaa_status.html @@ -188,7 +188,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" expected=Yes if [ "$result" == "SAM OK" ] ; then - [ $(echo "$siteLifeStatus" | grep -q "enabled" ; echo $?) -eq 0 ] && expected=No + if [ $(echo "$siteLifeStatus" | grep -q "enabled" ; echo $?) -eq 0 ] ; then + [ "x$siteDownStatus" == "xdowntime" ] || expected=No + fi fi sam_critical_host_tests=$(grep CRITICAL $THEPATH/out/cms_sam3_check.${thesite}.txt | awk '{print $2" "$4"
"}') if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then From 81de92314e7d19fd44ae48c44df6eb7290453798 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Wed, 4 Sep 2024 20:47:53 -0400 Subject: [PATCH 166/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 4a20d5d..dd94499 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -158,17 +158,17 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "" >> $THEPATH/site_aaa_status.html fi - thestring="Sites with Frequent Subscription Failure" - sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="Sites with Frequent Subscription Failure" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
Site
" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html thestring="" sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html - sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html sed -i "1s/^/
Sites with Frequent Subscription Failure
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
Sites with Frequent Subscription Failure
\n/" $THEPATH/site_aaa_status.html sed -i "1s/^/\n/" $THEPATH/site_aaa_status.html From 9c6d233e2a2c9a0c62a0a267ec0e588544632b12 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 5 Sep 2024 06:34:44 -0400 Subject: [PATCH 167/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index dd94499..7a80251 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -187,7 +187,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" expected=Yes - if [ "$result" == "SAM OK" ] ; then + if [ "$result" == "SAM3 OK" ] ; then if [ $(echo "$siteLifeStatus" | grep -q "enabled" ; echo $?) -eq 0 ] ; then [ "x$siteDownStatus" == "xdowntime" ] || expected=No fi From 3875cfdf39511782f8ba52404e6cd8b649bfa607 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 5 Sep 2024 16:35:38 -0400 Subject: [PATCH 168/192] Update create_fedmaps.py --- AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py index 1c188c5..55b7a70 100755 --- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -1342,7 +1342,8 @@ def getDNSARecords ( h ): isAlias = False aRecords = [] try : - result = dns.resolver.query(h, 'A') + #result = dns.resolver.query(h, 'A') + result = dns.resolver.resolve(h, 'A') for ipval in result: ip = ipval.to_text() (host, alias, ipl) = socket.gethostbyaddr(ip) From c4c1b6501e49ca1420f0e1ca7230e05bb9ada6b7 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 7 Sep 2024 05:42:27 -0400 Subject: [PATCH 169/192] Create getHostsFromAlias.py --- AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py diff --git a/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py b/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py new file mode 100644 index 0000000..d948040 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +import socket +import dns.resolver +h="xrootd-cms.infn.it" +result = dns.resolver.resolve(h, 'A') +for ipval in result: + ip = ipval.to_text() + (host, alias, ipl) = socket.gethostbyaddr(ip) + print ( host, alias, ipl ) From c5221968cab1b6671947cb9d1ffaad93ff6c3216 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 7 Sep 2024 05:53:20 -0400 Subject: [PATCH 170/192] Create .contacts --- AAAOps/FedProbeSendAAAMetrics/.contacts | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 AAAOps/FedProbeSendAAAMetrics/.contacts diff --git a/AAAOps/FedProbeSendAAAMetrics/.contacts b/AAAOps/FedProbeSendAAAMetrics/.contacts new file mode 100644 index 0000000..2a2f617 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/.contacts @@ -0,0 +1,6 @@ +xrootd-cms-redir-int.cr.cnaf.infn.it Daniel Spiga +xrootd-redic.pi.infn.it Daniel Spiga +llrxrd-redir.in2p3.fr Michael Mellin + +xrootd.unl.edu John Thiltges +cmsxrootd2.fnal.gov Yujun Wu From 12a406c04f0b1535bc94e78e8e409eae40838e50 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 7 Sep 2024 06:18:46 -0400 Subject: [PATCH 171/192] Update .contacts --- AAAOps/FedProbeSendAAAMetrics/.contacts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/.contacts b/AAAOps/FedProbeSendAAAMetrics/.contacts index 2a2f617..cccd032 100644 --- a/AAAOps/FedProbeSendAAAMetrics/.contacts +++ b/AAAOps/FedProbeSendAAAMetrics/.contacts @@ -1,6 +1,10 @@ -xrootd-cms-redir-int.cr.cnaf.infn.it Daniel Spiga -xrootd-redic.pi.infn.it Daniel Spiga -llrxrd-redir.in2p3.fr Michael Mellin +1 EU_Alias xrootd-cms.infn.it Daniele Spiga +1.1 EU xrootd-cms-redir-int.cr.cnaf.infn.it Daniele Spiga +1.2 EU xrootd-redic.pi.infn.it Daniele Spiga +1.3 EU llrxrd-redir.in2p3.fr Michael Mellin + +1 US_Alias cmsxrootd.fnal.gov Yujun Wu +1.1 US xrootd.unl.edu John Thiltges +1.2 US cmsxrootd2.fnal.gov Yujun Wu + -xrootd.unl.edu John Thiltges -cmsxrootd2.fnal.gov Yujun Wu From 770710ce86de1d834675928c1202f45e357ccf15 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 7 Sep 2024 06:19:03 -0400 Subject: [PATCH 172/192] Update getHostsFromAlias.py --- .../getHostsFromAlias.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py b/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py index d948040..3cb6735 100644 --- a/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py +++ b/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py @@ -1,9 +1,15 @@ #!/usr/bin/env python +import sys import socket import dns.resolver -h="xrootd-cms.infn.it" -result = dns.resolver.resolve(h, 'A') -for ipval in result: - ip = ipval.to_text() - (host, alias, ipl) = socket.gethostbyaddr(ip) - print ( host, alias, ipl ) + +if __name__ == "__main__": + h="xrootd-cms.infn.it" + h="cms-xrootd.gridpp.ac.uk" + if len(sys.argv) > 1 : + h=sys.argv[1] + result = dns.resolver.resolve(h, 'A') + for ipval in result: + ip = ipval.to_text() + (host, alias, ipl) = socket.gethostbyaddr(ip) + print ( host, alias, ipl ) From 31aa61892a5040726eb1bb6b9ecad5a34b3edba1 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sat, 7 Sep 2024 07:25:54 -0400 Subject: [PATCH 173/192] Update .contacts --- AAAOps/FedProbeSendAAAMetrics/.contacts | 35 +++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/.contacts b/AAAOps/FedProbeSendAAAMetrics/.contacts index cccd032..cd71ec4 100644 --- a/AAAOps/FedProbeSendAAAMetrics/.contacts +++ b/AAAOps/FedProbeSendAAAMetrics/.contacts @@ -1,10 +1,29 @@ -1 EU_Alias xrootd-cms.infn.it Daniele Spiga -1.1 EU xrootd-cms-redir-int.cr.cnaf.infn.it Daniele Spiga -1.2 EU xrootd-redic.pi.infn.it Daniele Spiga -1.3 EU llrxrd-redir.in2p3.fr Michael Mellin - -1 US_Alias cmsxrootd.fnal.gov Yujun Wu -1.1 US xrootd.unl.edu John Thiltges -1.2 US cmsxrootd2.fnal.gov Yujun Wu +1 EU_Alias xrootd-cms.infn.it Daniele Spiga T1_IT_CNAF +1.1 EU xrootd-cms-redir-int.cr.cnaf.infn.it Daniele Spiga T1_IT_CNAF +1.2 EU xrootd-redic.pi.infn.it Daniele Spiga* T2_IT_Pisa +1.3 EU llrxrd-redir.in2p3.fr Michael Mellin T1_FR_IN2P3 + 2 DE_Srv cmsxrootd-1.gridka.de Artur Il Darovic Gottmann* T1_DE_KIT + 2 DE_Srv cmsxrootd-2.gridka.de Artur Il Darovic Gottmann* T1_DE_KIT + 2 ES xrootd-es-pic.pic.es:1096 Antonio Perez-Calero Yzquierdo* T1_ES_PIC + 2 RU_Srv se-xrd01.jinr-t1.ru:1095 ? T1_RU_JINR + 2 UK cms-aaa-manager01.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2 UK xrootd-cms-uk.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2 UK xrootd-cms-uk.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2 UK_Alias xrootd.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2.1 UK xrootd01.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2.2 UK xrootd02.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2 UK_Alias cms-xrootd.gridpp.ac.uk Katy Ellis* T1_UK_London_IC + 2.1 UK xrootd01.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2.2 UK xrootd-cms-uk.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2.3 UK xrootd02.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + +1 US_Alias cmsxrootd.fnal.gov Yujun Wu T1_US_FNAL +1.1 US xrootd.unl.edu John Thiltges T2_US_Nebraska +1.2 US cmsxrootd2.fnal.gov Yujun Wu T1_US_FNAL + +? UK_Alias mover.pp.rl.ac.uk* Katy Ellis* T1_UK_RAL* + ?.1 UK heplnx230.pp.rl.ac.uk* Katy Ellis* T1_UK_RAL* + ?.2 UK heplnx231.pp.rl.ac.uk* Katy Ellis* T1_UK_RAL* + From bf2ab52de0ba8cbfd08785432c5b9deb7d41eeb2 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Mon, 9 Sep 2024 12:09:45 -0400 Subject: [PATCH 174/192] Update XRDFED-kibana-probe_JSON_General.py --- .../XRDFED-kibana-probe_JSON_General.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index 2c13ebf..f9d6285 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -336,10 +336,10 @@ def main(): 'file_below': FILEABOVE, 'file_above': FILEBELOW, 'extra_notes':CMSLINK}, - "XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', - 'file_below': FILEBELOW, - 'file_above': FILEABOVE, - 'extra_notes':CMSLINK}, + #"XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', + # 'file_below': FILEBELOW, + # 'file_above': FILEABOVE, + # 'extra_notes':CMSLINK}, "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', 'file_below': FILEBELOW, 'file_above': FILEABOVE, From 7c22a5987bcb118d2ecddebea8e1c419f1f38f99 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Mon, 9 Sep 2024 12:49:46 -0400 Subject: [PATCH 175/192] Update XRDFED-kibana-probe_JSON_General.py --- .../XRDFED-kibana-probe_JSON_General.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py index f9d6285..39bb6e8 100644 --- a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -46,8 +46,8 @@ def clear_lock(): except Exception as e: print ("could not remove lockfile:"+str(e)) def env_setup(): - os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' - os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' + #os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' + #os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' #os.environ['X509_USER_PROXY']='/root/.globus/.proxy' if 'ufhpc' in socket.gethostname() : os.environ['X509_USER_PROXY']=os.environ['HOME']+'/.cmsuser.proxy' From 3648ad6a66c3b62a7cdbde0a09d8d7ad02c2f3e7 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 09:20:23 -0400 Subject: [PATCH 176/192] Update probe_create_send_aaa_metrics.sh --- .../probe_create_send_aaa_metrics.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 7a80251..4bb523d 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -239,7 +239,16 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then fi echo "Sites subscribed to the Production Federation: " $nprod fi - +ip_no_host=$(echo $(grep [0-9]: $THEPATH/out/xrdmapc_prod_* | awk '{print $NF}' | sort -u)) +if [ "x$ip_no_host" != "x" ] ; then + ( + echo "To: $notifytowhom" + echo "Subject: $(basename $0) on $(/bin/hostname -s) IPs without Hostnames" + echo "Reply-to: noreply@cern.ch" + echo "Content-Type: text/html" + for ip in $ip_no_host ; do echo $ip"
" ; done + ) | /usr/sbin/sendmail -t +fi printf "$(/bin/hostname -s) $(basename $0) \nxrdmapc errors:\n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" [ -f ${THELOG}_$(date +%Y%m%d) ] || touch ${THELOG}_$(date +%Y%m%d) From d55d4be8cb801e16587b8339f7b98f5883324e81 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 09:45:27 -0400 Subject: [PATCH 177/192] Create siteLifeStatus.sh --- AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh new file mode 100644 index 0000000..c35e9b8 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh @@ -0,0 +1,9 @@ +#!/bin/bash +if [ $# -lt 1 ] ; then + echo ERROR: $(basename $0) TX__YY_ZZZZ + exit 1 +fi +site=$1 +THEPATH=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +export GRAFANA_VIEWER_TOKEN=$(echo $(cat $THEPATH/token.txt)) +python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $site From 67c67487c3a756a6fd1100bb6aededd582435a13 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 09:57:15 -0400 Subject: [PATCH 178/192] Update siteLifeStatus.sh --- AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh index c35e9b8..30a12b0 100644 --- a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh @@ -3,7 +3,14 @@ if [ $# -lt 1 ] ; then echo ERROR: $(basename $0) TX__YY_ZZZZ exit 1 fi -site=$1 +sites=$1 THEPATH=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics export GRAFANA_VIEWER_TOKEN=$(echo $(cat $THEPATH/token.txt)) -python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $site +if [ $(echo $sites | grep -q T ; echo $?) -eq 0 ] ; then + python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $sites +else + for site in $(echo /cvmfs/cms.cern.ch/SITECONF/T[1-2]* ) ; do + s=$(echo $site | cut -d/ -f5) + echo $s LifeStatus=$(python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $s 2>/dev/null ) + done +fi From 06eec1187e35db6ab1401bdf007beaba934d8d47 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 16:12:35 -0400 Subject: [PATCH 179/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 4bb523d..43e3c54 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -173,7 +173,6 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then sed -i "1s/^/\n/" $THEPATH/site_aaa_status.html if [ ! -f $THEPATH/site_aaa_status.txt ] ; then - echo "" >> $THEPATH/site_aaa_status.html printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt fi sam3result= @@ -201,7 +200,8 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html #echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html - echo "" >> $THEPATH/site_aaa_status.html + other_comments=$(echo $(grep "|" $THEPATH/site_aaa_status.txt | cut -d\| -f2-)) + echo "" >> $THEPATH/site_aaa_status.html else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt echo "" >> $THEPATH/site_aaa_status.html From d640994ea4d2fdf476e0075af06678dac2d690cb Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 16:25:41 -0400 Subject: [PATCH 180/192] Update probe_create_send_aaa_metrics.sh --- .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 43e3c54..4fd60d3 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -196,11 +196,11 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') WkCount=$(expr $WkCount_previous + 1) + other_comments=$(echo $(grep "|" $THEPATH/site_aaa_status.txt | grep $thesite | cut -d\| -f2-)) sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt - printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected >> $THEPATH/site_aaa_status.txt + printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected "|" $other_comments >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html #echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html - other_comments=$(echo $(grep "|" $THEPATH/site_aaa_status.txt | cut -d\| -f2-)) echo "" >> $THEPATH/site_aaa_status.html else printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt From 0b48affc279a841ebd004f850aec4f0b77550cd8 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 16:31:14 -0400 Subject: [PATCH 181/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 4fd60d3..25636c1 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -196,7 +196,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') WkCount=$(expr $WkCount_previous + 1) - other_comments=$(echo $(grep "|" $THEPATH/site_aaa_status.txt | grep $thesite | cut -d\| -f2-)) + other_comments=$(echo $(grep "$thesite" $THEPATH/site_aaa_status.txt | grep "|" | cut -d\| -f2-)) sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected "|" $other_comments >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html From 561783f1a8926a8955b3fb552ac0a35fcd55240a Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 16:34:56 -0400 Subject: [PATCH 182/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 25636c1..83e0c84 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -197,8 +197,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') WkCount=$(expr $WkCount_previous + 1) other_comments=$(echo $(grep "$thesite" $THEPATH/site_aaa_status.txt | grep "|" | cut -d\| -f2-)) + [ "x$other_comments" == "x" ] || other_comments=" | $other_comments" sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt - printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected "|" $other_comments >> $THEPATH/site_aaa_status.txt + printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected $other_comments >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html #echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html From 16c4b45c60090c3e74b0535ba885c7b1ef700ad0 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 17:52:28 -0400 Subject: [PATCH 183/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 83e0c84..d35e99d 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -192,6 +192,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then fi fi sam_critical_host_tests=$(grep CRITICAL $THEPATH/out/cms_sam3_check.${thesite}.txt | awk '{print $2" "$4"
"}') + other_comments= if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') From 965b7a3057c297e0e144314e20647285542dd3be Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Thu, 19 Sep 2024 18:31:50 -0400 Subject: [PATCH 184/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index d35e99d..4a923a9 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -200,7 +200,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then other_comments=$(echo $(grep "$thesite" $THEPATH/site_aaa_status.txt | grep "|" | cut -d\| -f2-)) [ "x$other_comments" == "x" ] || other_comments=" | $other_comments" sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt - printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected $other_comments >> $THEPATH/site_aaa_status.txt + printf "%20s %20s %20s %7s %7s %80s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected "$other_comments" >> $THEPATH/site_aaa_status.txt sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html #echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html echo "" >> $THEPATH/site_aaa_status.html From a1fe2ab7b8151409a0cae9bbebf35c30d4638b04 Mon Sep 17 00:00:00 2001 From: Bockjoo Kim Date: Sun, 13 Oct 2024 14:32:32 -0400 Subject: [PATCH 185/192] Update probe_create_send_aaa_metrics.sh --- AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh index 4a923a9..c7835db 100755 --- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -130,6 +130,8 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then echo "Reply-To: noreply@cern.ch" echo "Subject: XRootD Version Role List" echo "Content-Type: text/html" + echo "AAA Subscription: https://monit-grafana.cern.ch/d/5njhdTrWk/site-subscription?orgId=11
" + echo "" echo "
"
 	 echo $THEPATH/create_fedmaps.log
 	 cat $THEPATH/create_fedmaps.log

From b63a7b1b0612627a6586a5b99e4aec0393e53d2e Mon Sep 17 00:00:00 2001
From: Bockjoo Kim 
Date: Sun, 13 Oct 2024 14:33:56 -0400
Subject: [PATCH 186/192] Update probe_create_send_aaa_metrics.sh

---
 AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
index c7835db..b171146 100755
--- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
+++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
@@ -131,7 +131,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then
          echo "Subject: XRootD Version Role List"
          echo "Content-Type: text/html"
 	 echo "AAA Subscription: https://monit-grafana.cern.ch/d/5njhdTrWk/site-subscription?orgId=11 
"
-  	 echo ""
+  	 echo "Redirector Availability: https://monit-grafana.cern.ch/d/serviceAvailability/overview-service-availability?orgId=11&var-category=All&from=now-24h&to=now-15m%2Fm 
"
          echo "
"
 	 echo $THEPATH/create_fedmaps.log
 	 cat $THEPATH/create_fedmaps.log

From 98fe799ba497712fba68a7e95be145a88efa301d Mon Sep 17 00:00:00 2001
From: Bockjoo Kim 
Date: Thu, 17 Oct 2024 17:16:41 -0400
Subject: [PATCH 187/192] Update probe_create_send_aaa_metrics.sh

---
 .../probe_create_send_aaa_metrics.sh                     | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
index b171146..8eebf44 100755
--- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
+++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
@@ -240,6 +240,15 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then
       #   printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom 
       #fi
       #exit 1
+   else
+      echo INFO not "($nprod_exp -gt $nprod)"
+      (
+         echo "To: $notifytowhom"
+         echo "Subject: $(basename $0) on $(/bin/hostname -s) $nprod_exp is not gt $nprod "
+	 echo "Reply-to: noreply@cern.ch"
+         echo "Content-Type: text/html"
+	 $nprod_exp is not greater than $nprod	 
+       )  | /usr/sbin/sendmail -t
    fi
    echo "Sites subscribed to the Production Federation: " $nprod
 fi

From f17066ff3b3c82b7be6a122794f8604417859d0c Mon Sep 17 00:00:00 2001
From: Bockjoo Kim 
Date: Thu, 17 Oct 2024 19:20:57 -0400
Subject: [PATCH 188/192] Update siteLifeStatus.py

---
 AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py
index 3b44766..50a7bac 100644
--- a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py
+++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py
@@ -85,6 +85,7 @@ def siteLifeStatus (site='T2_US_Florida', what='sts15min', dbid=9980, gte="now-2
         siteStatus[hit['_source']['data']['name']] = hit['_source']['data']['status']
     if len(siteStatus.keys()) == 0 :
         return 'UNKNOWN'
+    if not site in siteStatus.keys() : return 'NoKey'
     return siteStatus[site]
 
 if __name__ == "__main__":

From 009056fccfd335374b1ee5c17a64e654eca52fcd Mon Sep 17 00:00:00 2001
From: Bockjoo Kim 
Date: Thu, 17 Oct 2024 19:25:51 -0400
Subject: [PATCH 189/192] Update probe_create_send_aaa_metrics.sh

---
 AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
index 8eebf44..1d40ad6 100755
--- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
+++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
@@ -143,6 +143,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then
    fi
    printf "$subscribed_sites\n" > $THEPATH/subscribed_sites_$nprod.txt
    if [ $nprod_exp -gt $nprod ] ; then
+      echo DEBUG nprod_exp -gt $nprod : $nprod_exp -gt $nprod
       thediff=
       if [ -f $THEPATH/subscribed_sites_$nprod_exp.txt ] ; then
          thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2 | sort -u)

From 1039fe701f761088f92a635b67149735f7ab8b70 Mon Sep 17 00:00:00 2001
From: Bockjoo Kim 
Date: Fri, 18 Oct 2024 04:49:56 -0400
Subject: [PATCH 190/192] Update probe_create_send_aaa_metrics.sh

---
 .../FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
index 1d40ad6..816f876 100755
--- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
+++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
@@ -143,7 +143,7 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then
    fi
    printf "$subscribed_sites\n" > $THEPATH/subscribed_sites_$nprod.txt
    if [ $nprod_exp -gt $nprod ] ; then
-      echo DEBUG nprod_exp -gt $nprod : $nprod_exp -gt $nprod
+      echo DEBUG nprod_exp -gt nprod : $nprod_exp -gt $nprod
       thediff=
       if [ -f $THEPATH/subscribed_sites_$nprod_exp.txt ] ; then
          thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2 | sort -u)
@@ -227,6 +227,8 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then
       #fi
       #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then
       #echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0)  \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom
+      echo DEBUG sam3result
+      printf "$sam3result\n"
       if [ $(echo $sam3result | grep -q "SAM3 OK" ; echo $?) -eq 0 ] ; then
        (
          echo "To: $notifytowhom"

From f20f1cbba2535c5819f6361643973937e7f16e18 Mon Sep 17 00:00:00 2001
From: Bockjoo Kim 
Date: Fri, 18 Oct 2024 07:23:31 -0400
Subject: [PATCH 191/192] Update probe_create_send_aaa_metrics.sh

---
 AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
index 816f876..dbd4453 100755
--- a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
+++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh
@@ -237,6 +237,9 @@ if [ -f $THEPATH/check_subscribed_sites.sh ] ; then
          echo "Content-Type: text/html"
 	 cat $THEPATH/site_aaa_status.html	 
        )  | /usr/sbin/sendmail -t
+       echo DEBUG sam3result OK mail sent
+      else
+       echo DEBUG sam3result not OK
       fi
       #else
       #   echo $sam3result | grep -q "SAM3 OK" && \

From 73a7543fe4735d14ce38d30c6b3c021b1af653fa Mon Sep 17 00:00:00 2001
From: Bockjoo Kim 
Date: Tue, 9 Dec 2025 13:05:34 -0500
Subject: [PATCH 192/192] Update create_fedmaps.py

timelimit increased for xrd_info with the xrdfs command
---
 AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py
index 55b7a70..7b84e0a 100755
--- a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py
+++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py
@@ -97,7 +97,7 @@ def xrd_info(redirector,what):
     config_out = "unknown"
     count = 0
     count_limit = 3
-    timelimit = 2
+    timelimit = 180 # 2
     if 'xrdmapc' in command :
        timelimit = 180
        count_limit = 1
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
$thesite $siteLifeStatus $result $siteDownStatus $WkCount $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result $siteDownStatus $WkCount $expected $sam_critical_host_tests $other_comments
$thesite $siteLifeStatus $result $siteDownStatus 1 $expected $sam_critical_host_tests
$thesite $siteLifeStatus $result $siteDownStatus $WkCount $expected $sam_critical_host_tests $other_comments
$thesite $siteLifeStatus $result $siteDownStatus $WkCount $expected $sam_critical_host_tests $other_comments
$thesite $siteLifeStatus $result $siteDownStatus $WkCount $expected $sam_critical_host_tests $other_comments