diff --git a/.gitignore b/.gitignore index ba76c66..03e7ade 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ -works/ +AAAOps/Federation/in/ +AAAOps/Federation/out/ +AAAOps/tools/timestamp.sh +AAAOps/Federation/hostIPv4.txt +AAAOps/Federation/transit-hostIPv4.txt +hostIPv4.txt +transit-hostIPv4.txt diff --git a/AAAOps/FedProbeSendAAAMetrics/.contacts b/AAAOps/FedProbeSendAAAMetrics/.contacts new file mode 100644 index 0000000..cd71ec4 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/.contacts @@ -0,0 +1,29 @@ +1 EU_Alias xrootd-cms.infn.it Daniele Spiga T1_IT_CNAF +1.1 EU xrootd-cms-redir-int.cr.cnaf.infn.it Daniele Spiga T1_IT_CNAF +1.2 EU xrootd-redic.pi.infn.it Daniele Spiga* T2_IT_Pisa +1.3 EU llrxrd-redir.in2p3.fr Michael Mellin T1_FR_IN2P3 + 2 DE_Srv cmsxrootd-1.gridka.de Artur Il Darovic Gottmann* T1_DE_KIT + 2 DE_Srv cmsxrootd-2.gridka.de Artur Il Darovic Gottmann* T1_DE_KIT + 2 ES xrootd-es-pic.pic.es:1096 Antonio Perez-Calero Yzquierdo* T1_ES_PIC + 2 RU_Srv se-xrd01.jinr-t1.ru:1095 ? T1_RU_JINR + 2 UK cms-aaa-manager01.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2 UK xrootd-cms-uk.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2 UK xrootd-cms-uk.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2 UK_Alias xrootd.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2.1 UK xrootd01.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2.2 UK xrootd02.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2 UK_Alias cms-xrootd.gridpp.ac.uk Katy Ellis* T1_UK_London_IC + 2.1 UK xrootd01.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + 2.2 UK xrootd-cms-uk.gridpp.rl.ac.uk Katy Ellis* T1_UK_RAL + 2.3 UK xrootd02.grid.hep.ph.ic.ac.uk Katy Ellis* T2_UK_London_IC + +1 US_Alias cmsxrootd.fnal.gov Yujun Wu T1_US_FNAL +1.1 US xrootd.unl.edu John Thiltges T2_US_Nebraska +1.2 US cmsxrootd2.fnal.gov Yujun Wu T1_US_FNAL + +? UK_Alias mover.pp.rl.ac.uk* Katy Ellis* T1_UK_RAL* + ?.1 UK heplnx230.pp.rl.ac.uk* Katy Ellis* T1_UK_RAL* + ?.2 UK heplnx231.pp.rl.ac.uk* Katy Ellis* T1_UK_RAL* + + + diff --git a/AAAOps/FedProbeSendAAAMetrics/README b/AAAOps/FedProbeSendAAAMetrics/README new file mode 100644 index 0000000..61121a3 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/README @@ -0,0 +1,59 @@ +[0] How to Check Things + 1 To check the hostname the ip from an xrdmapc belongs to, for example, run + +xrdfs [2001:e10:4000:122::105]:11001 query config sitename + +[1] Essential Files and Directories +create_fedmaps.py +aaa_federation.py +probe_create_send_aaa_metrics.sh +CMSMonitoring +credentials.json + +[2] Needed Directories Created +out +logs + +[3] Utility Scripts +check_timeout.sh +compare_timeout_list.sh +mapHostSitename.sh +listSrv.sh +check_xrdmapc_from_vofeed.sh +check_vofeed_from_xrdmapc.sh +check_upstream_redirector_examples.sh +check_subscribed_sites.sh + +[4] Installing +# First check the needed python modules +python3 -c "import numpy" +python3 -c "import stomp" +python3 -c "import dns.resolver" +# 1 +cd /opt +git clone git@github.com:CMSCompOps/TransferTeam.git +cd TransferTeam/AAAOps/FedProbeSendAAAMetrics +cp ~/credentials.json ./ +# OR +# 2 +cd /opt/TransferTeam +git pull origin master +cd AAAOps/FedProbeSendAAAMetrics +cp ~/credentials.json ./ +git clone https://github.com/dmwm/CMSMonitoring.git +crontab -l > crontab +echo '14,29,44,59 * * * * /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh > /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/logs/probe_create_send_aaa_metrics.log 2>&1' >> crontab +# Add the cron to the gitlab + +[5] Development +On cmsio3 as bockjoo +cd /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/ +git add create_fedmaps.py +git status +git commit -m "create_fedmaps.py is updated" +git push origin master + +On vocms036.cern.ch as root +cd /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/ +git pull origin master +# if merge conflicts, git checkout --theirs . diff --git a/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py new file mode 100755 index 0000000..5479e54 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/aaa_federation.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +import re +import sys + +#RHEL8 default is 7.0.0 __requires__ = 'stomp.py==4.1.21' +__requires__ = 'stomp.py==7.0.0' + +from pkg_resources import load_entry_point +import stomp + +import os +import subprocess +import urllib +import json +from datetime import datetime +import collections +import stomp +import argparse +from CMSMonitoring.StompAMQ import StompAMQ +import uuid +from itertools import islice +import hashlib +from uuid import uuid4 +from urllib.request import urlopen +import time +#RHEL8 +# CMSMonitoring modules +try: + from CMSMonitoring.StompAMQ7 import StompAMQ7 +except ImportError: + print("ERROR: Could not import StompAMQ7") + sys.exit(1) + +print("stomp version", stomp.__version__) + +FedProbeSendAAAMetrics = '/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' +FedProbeSendAAAMetrics = '/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' + +class OptionParser(): + def __init__(self): + "User based option parser" + self.parser = argparse.ArgumentParser() + msg = "Send aaa subscription json file via StompAMQ to a broker, provide broker credentials in JSON file" + self.parser.add_argument("--amq", action="store", + dest="amq", default="", help=msg, required=True) + #self.parser.add_argument("--create_json_only", action="store", + # dest="create_json_only", default="", help=msg, required=False) + + +class MyListener(stomp.ConnectionListener): + def on_error(self, headers, message): + print('received an error "%s"' % message) + + def on_message(self, headers, message): + print('received a message "%s"' % message) + + +def credentials(fname=None): + "Read credentials from MONIT_BROKER environment" + if not fname: + fname = os.environ.get('MONIT_BROKER', '') + if not os.path.isfile(fname): + raise Exception( + "Unable to locate MONIT credentials, please setup MONIT_BROKER") + return {} + with open(fname, 'r') as istream: + data = json.load(istream) + return data + +def parseFederationJson(): + ''' + modifies federation.json with tags and + Return json object to feed cern metris + ''' + #url = 'http://vocms039.cern.ch/aaa-fedinfo/federations.json' + #result = json.load(urlopen(url)) + #print ("open f") + with open(FedProbeSendAAAMetrics+'out/federations.json') as f: url = f.read() + #print ("url ",url) + result = json.loads(url) + + if result['prod'] and result['trans']: + fed_data = createFlattenJson(result) + #print (opts.create_json_only) + f = open(FedProbeSendAAAMetrics+"fed.json", "r") + #if opts.create_json_only : + # print (" 2 ",opts.create_json_only) + # for x in f: + # data = json.loads(x) + #else: + for x in f: + data = json.loads(x) + #print ( data ) + yield data + +def createFlattenJson(json_file): + ''' + flattens the federation.json file + ''' + #print ("createFlattenJson") + for k,v in json_file.items(): + #print ( k, v.keys() ) + #for dic in v: + # print ( k, dic.keys() ) + #break + if ("prod" in k): + statusCode = 1 + stateName = "Production Federation" + elif ("trans" in k): + statusCode = 2 + stateName = "Transitional Federation" + else: + statusCode = 3 + stateName = "not at any federation" + for site in v: + #print ( k, site ) + #my_json = ('{"siteName" : "%s", "state" : "%s", "statusCode" : %s, "stateName" : "%s"}' %(site, k, statusCode, stateName)) + #for isite in range(len(site['sites'])): + # if 'XROOTD' not in site['flavors'][isite] : continue + # #contact = site['contact'][0] + # ##print (site['sites'][isite], " contact len ",len(site['contact'])) + # #if not contact : contact = site['contact'][(len(site['contact']) - 1)] + # #print (" s = ",site['sites'][isite], site['flavors'][isite], site['endpoints'][isite], site['federation'][isite]) #print (" site = ",site['sites']) + my_json = ('{"siteName" : "%s", "state" : "%s", "statusCode" : %s, "stateName" : "%s", "flavor" : "%s", "endpoint" : "%s", "xrootd_version" : "%s", "xrootd_role" : "%s", "storage" : "%s", "contact" : "%s"}' %(site['sites'], k, statusCode, stateName, site['flavors'], site['endpoints'], site['xrootd_version'], site['xrootd_role'], site['xrootd_storage'], site['contact'])) + with open(FedProbeSendAAAMetrics+"fed.json","a") as f: + f.write("%s\n" %my_json) + +if __name__ == "__main__": + #RHEL8 + ts = int(time.time()) * 1000 + #RHEL8 + optmgr = OptionParser() + opts = optmgr.parser.parse_args() + payload = parseFederationJson() + #for ipay in payload: #bockjoo comment this out when in production + # print (ipay) #bockjoo comment this out when in production + ##print ("Check "+FedProbeSendAAAMetrics+"fed.json") + #sys.exit(0) # bockjoo to uncomment + if opts.amq: + creds = credentials(opts.amq) + username = creds['username'] + password = creds['password'] + topic = creds['topic'] + host, port = creds['host_and_ports'].split(':') + port = int(port) + producer = creds['producer'] + topic = creds['topic'] + hosts = [(host, port)] + #RHEL8 + doc_type = creds['type'] + #RHEL8 if creds and StompAMQ: + #RHEL8 amq = StompAMQ(username, password, producer, topic, key=None, + #RHEL8 cert=None, validation_schema=None, host_and_ports=[(host, port)]) + if creds and StompAMQ7: + amq = StompAMQ7(username, password, producer, topic, key=None, + cert=None, validation_schema=None, host_and_ports=[(host, port)]) + eod = False + wait_seconds = 10 + while not eod: + messages = [] + for d in payload: + #RHEL8 notif,_,_ = amq.make_notification(d, "aaa_federations_document", dataSubfield=None) + notif,_,_ = amq.make_notification(payload=d, doc_type=doc_type, + producer=producer, ts=ts) + messages.append(notif) + if messages: + print(messages) + amq.send(messages) + time.sleep(wait_seconds) + else: + eod = True diff --git a/AAAOps/FedProbeSendAAAMetrics/check_subscribed_sites.sh b/AAAOps/FedProbeSendAAAMetrics/check_subscribed_sites.sh new file mode 100755 index 0000000..4f799bc --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_subscribed_sites.sh @@ -0,0 +1,7 @@ +#!/bin/bash +what=Production +[ $# -gt 0 ] && what=$1 +sites=$(grep "$what" /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/fed.json | awk '{print $3}' | sort -u) +printf "$sites\n" +echo "Count: $(echo $sites | wc -w)" + diff --git a/AAAOps/FedProbeSendAAAMetrics/check_timeout.sh b/AAAOps/FedProbeSendAAAMetrics/check_timeout.sh new file mode 100755 index 0000000..c3a676e --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_timeout.sh @@ -0,0 +1,18 @@ +#!/bin/bash +endpoints=$(grep ^T[0-9]_ create_fedmaps.log | awk '{print $2}' | sort -u) +for e in $endpoints ; do + #echo $e + log=tmp/$(echo $e | sed 's#:##g').log + xrdfs $e query config version > $log 2>&1 & + theps=$! + i=0 + while [ $i -lt 30 ] ; do + + ps auxww | grep -v grep | grep $theps | awk '{print "+"$2"+"}' | grep -q "+"$theps"+" + + [ $? -ne 0 ] && break + i=$(expr $i + 1) + sleep 1 + done + echo $i $e $(cat $log) +done diff --git a/AAAOps/FedProbeSendAAAMetrics/check_upstream_redirector_examples.sh b/AAAOps/FedProbeSendAAAMetrics/check_upstream_redirector_examples.sh new file mode 100644 index 0000000..b78d0a0 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_upstream_redirector_examples.sh @@ -0,0 +1,6 @@ +#!/bin/bash +n=1000 ; xrdmapc --list all cms-xrd-global01.cern.ch:1094 2>/dev/null | grep -A $n ^1 | grep -B $n kr | grep "^1\|cms-t2-se01.sdfarm.kr" +n=1000 ; xrdmapc --list all cms-xrd-global.cern.ch:1094 2>/dev/null | grep -A $n ^1 | grep -B $n xrootd-cms-01.cr.cnaf.infn.it:0 | grep "^1\|xrootd-cms-01.cr.cnaf.infn.it:0" +n=1000 ; xrdmapc --list all cms-xrd-global.cern.ch:1094 2>/dev/null | grep -A $n ^2 | grep -B $n xrootd-cms-01.cr.cnaf.infn.it:0 | grep "^2\|xrootd-cms-01.cr.cnaf.infn.it:0" +n=1000 ; xrdmapc --list all cms-xrd-global.cern.ch:1094 2>/dev/null | grep -A $n ^2 | grep -B $n xrootd-cms-01.cr.cnaf.infn.it:0 | grep -B 1 "^2\|xrootd-cms-01.cr.cnaf.infn.it:0" + diff --git a/AAAOps/FedProbeSendAAAMetrics/check_vofeed_from_xrdmapc.sh b/AAAOps/FedProbeSendAAAMetrics/check_vofeed_from_xrdmapc.sh new file mode 100755 index 0000000..577aba5 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_vofeed_from_xrdmapc.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Checks all xrootd endpoint in vofeed https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml is found from xrdmapcall +# +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +#xredirs=$(cat $FedProbeSendAAAMetrics/fed.json | cut -d, -f6 | sort -u | cut -d\" -f4 | cut -d: -f1) +#xredirs=$(grep " Man " $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt | awk '{print $NF}' | cut -d: -f1 | sort -u) +xredirs=$(grep " Man " $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt | cut -d: -f2- | awk '{print $3}' | sort -u) +output=$(printf "%5s %60s %5s\n" Found Xredirtor MLevel) +for xredir in $xredirs ; do + MLevel=$(echo $(grep $xredir $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt | cut -d: -f2- | awk '{print $1}' | sort -u) | sed 's# #+#g') + Found=$(grep -q $xredir $FedProbeSendAAAMetrics/fed.json ; echo $?) + output="$output\n"$(printf "%5s %60s %5s\n" $Found $xredir $MLevel) + # + # Analyze the outcome + # + # 1 Found=1 MLevel!=1 -> 1 $xredir can be one of Found=1 MLevel=1 + # 2 $xredir could be found from the alias in vofeed + +done +printf "$output\n" +exit 0 + +Found Xredirtor MLevel + 0 cms-aaa-manager01.gridpp.rl.ac.uk:1094 2+3 + 0 cmsio7.rc.ufl.edu:1094 2 + 1 cmsxrootd2.fnal.gov:1094 1 + 1 cmsxrootd.ihep.ac.cn:1094 2 + 0 cmsxrootd-site1.fnal.gov:1094 2 + 0 cmsxrootd-site2.fnal.gov:1094 2 + 0 cmsxrootd-site3.fnal.gov:1094 2 + 1 dmsdcatst03.fnal.gov:1094 2 + 1 llrxrd-redir.in2p3.fr:1094 1 + 1 llrxrd-redir.in2p3.fr:1194 2 + 1 llrxrd-redir.in2p3.fr:1294 2 + 0 osg-se.sprace.org.br:1094 2 + 0 pubxrootd.hep.wisc.edu:1094 2 + 0 redirector.t2.ucsd.edu:1094 2 + 0 xroot02.ncg.ingrid.pt:1094 2 + 1 xrootd01.grid.hep.ph.ic.ac.uk:1094 1 + 1 xrootd.ba.infn.it:1094 1 + 1 xrootd.ba.infn.it:1194 2 + 1 xrootd.ba.infn.it:1294 2 + 1 xrootd-cms-01.cr.cnaf.infn.it:0 3 + 0 xrootd-cms-01.cr.cnaf.infn.it:1094 3 + 0 xrootd-cms-02.cr.cnaf.infn.it:1094 3 + 0 xrootd.cmsaf.mit.edu:1094 2 + 1 xrootd-cms-redir-int.cr.cnaf.infn.it:1094 1 + 1 xrootd-cms-redir-int.cr.cnaf.infn.it:1194 2 + 1 xrootd-cms-redir-int.cr.cnaf.infn.it:1294 2 + 1 xrootd-cms-uk.gridpp.rl.ac.uk:1094 2 https://ggus.eu/index.php?mode=ticket_info&ticket_id=153496&come_from=submit + 1 xrootd-es-cie.ciemat.es:1096 2 + 1 xrootd-es-pic.pic.es:1096 2 + 0 xrootd.hepgrid.uerj.br:1094 2 + 0 xrootd.hep.kbfi.ee:1094 2 + 0 xrootd-local.unl.edu:1094 2 + 0 xrootd.rcac.purdue.edu:1094 2 + 1 xrootd-redic.pi.infn.it:1094 1 + 1 xrootd-redic.pi.infn.it:1194 2 + 1 xrootd-redic.pi.infn.it:1294 2 + 1 xrootd-redir1-vanderbilt.sites.opensciencegrid.org:1094 2 + 1 xrootd-redir2-vanderbilt.sites.opensciencegrid.org:1094 2 + 0 xrootd-redir.ultralight.org:1094 2 diff --git a/AAAOps/FedProbeSendAAAMetrics/check_xrdmapc_from_vofeed.sh b/AAAOps/FedProbeSendAAAMetrics/check_xrdmapc_from_vofeed.sh new file mode 100755 index 0000000..fa44229 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/check_xrdmapc_from_vofeed.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# +# Checks all xrootd endpoint in vofeed https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml is found from xrdmapcall +# +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +FedProbeSendAAAMetrics=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +xredirs=$(cat $FedProbeSendAAAMetrics/fed.json | cut -d, -f6 | sort -u | cut -d\" -f4 | cut -d: -f1) +output=$(printf "%5s %50s %3s\n" Found Xredirtor Tier) +for xredir in $xredirs ; do + Tier=$(echo $(grep $xredir $FedProbeSendAAAMetrics/fed.json | cut -d\" -f4 | cut -d_ -f1 | sed 's#T##g' | sort -u) | sed 's# #+#g') + Found=$(grep -q $xredir $FedProbeSendAAAMetrics/out/xrdmapc_all_0.txt $FedProbeSendAAAMetrics/out/xrdmapc_all_1.txt ; echo $?) + output="$output\n"$(printf "%5s %50s %3s\n" $Found $xredir $Tier) +done +printf "$output\n" +exit 0 +Found Xredirtor Tier + 1 bonner04.rice.edu 3 + 1 brux11.hep.brown.edu 3 + 0 ccsrm.ihep.ac.cn 2 + 1 ccxrdcms.in2p3.fr 1 + 0 ceph-gw10.gridpp.rl.ac.uk 1 + 0 ceph-gw11.gridpp.rl.ac.uk 1 + 1 cluster142.knu.ac.kr 3 + 0 cms03.lcg.cscs.ch 2 + 0 cms-aaa-manager01.gridpp.rl.ac.uk 1 + 0 cmsdata.phys.cmu.edu 3 + 0 cmsio7.rc.ufl.edu 2 + 0 cmsrm-xrootd01.roma1.infn.it 2 + 0 cmsrm-xrootd02.roma1.infn.it 2 + 0 cms-se0.kipt.kharkov.ua 2 + 1 cms-se.hep.uprm.edu 3 + 0 cms-t2-se01.sdfarm.kr 2 + 1 cmsxrd.ts.infn.it 3 + 1 cmsxrootd-redirectors.gridka.de 1 + 0 cmsxrootd-site1.fnal.gov 1 + 0 cmsxrootd-site2.fnal.gov 1 + 0 cmsxrootd-site3.fnal.gov 1 + 0 dc2-grid-64.brunel.ac.uk 2 + 1 dcache-cms-xrootd.desy.de 2 + 1 eos.grid.vbc.ac.at 2 + 0 eymir.grid.metu.edu.tr 2 + 0 gaexroot01.ciemat.es 2 + 0 gfe02.grid.hep.ph.ic.ac.uk 2 + 0 grid02.physics.uoi.gr 2 + 0 grid143.kfki.hu 2 + 1 grid71.phy.ncu.edu.tw 3 + 0 grid-dcache.physik.rwth-aachen.de 2 + 0 grse001.inr.troitsk.ru 2 + 0 hask.csc.fi 2 + 1 hepcms-0.umd.edu 3 + 0 heplnx228.pp.rl.ac.uk 2 + 0 heplnx229.pp.rl.ac.uk 2 + 1 hepxrd01.colorado.edu 3 + 1 ingrid-se03.cism.ucl.ac.be 2 + 0 ingrid-se04.cism.ucl.ac.be 2 + 1 ingrid-se05.cism.ucl.ac.be 2 + 0 ingrid-se06.cism.ucl.ac.be 2 + 1 kodiak-se.baylor.edu 3 + 0 lcgse01.phy.bris.ac.uk 2 + 0 lcgsexrd.jinr.ru 2 + 1 llrpp01.in2p3.fr 2 + 1 lyoeos.in2p3.fr 3 + 1 lyogrid06.in2p3.fr 3 + 0 maite.iihe.ac.be 2 + 0 node12.datagrid.cea.fr 2 + 0 nute.csc.fi 2 + 0 osg-se.sprace.org.br 2 + 1 pcncp22.ncp.edu.pk 2 + 0 polgrid4.in2p3.fr 2 + 0 pool01.ifca.es 2 + 0 pool02.ifca.es 2 + 0 pool03.ifca.es 2 + 0 pool04.ifca.es 2 + 0 pool05.ifca.es 2 + 0 pool06.ifca.es 2 + 0 pool07.ifca.es 2 + 0 pool08.ifca.es 2 + 0 pubxrootd.hep.wisc.edu 2 + 0 redirector.t2.ucsd.edu 2 + 0 sbgse1.in2p3.fr 2 + 0 se01.grid.nchc.org.tw 2 + 0 se3.itep.ru 2 + 0 se.cis.gov.pl 2 + 0 se-xrd01.jinr-t1.ru 1 + 0 ss-01.recas.ba.infn.it 2 + 0 ss-02.recas.ba.infn.it 2 + 0 ss-03.recas.ba.infn.it 2 + 0 stormgf1.pi.infn.it 2 + 1 stormgf2.pi.infn.it 2 + 1 stormgf3.pi.infn.it 2 + 1 storm.mib.infn.it 3 + 0 t2-cms-xrootd01.desy.de 2 + 0 t2-cms-xrootd02.desy.de 2 + 0 t2-cms-xrootd03.desy.de 2 + 0 t2-xrdcms.lnl.infn.it 2 + 1 t3se01.psi.ch 3 + 0 vobox0002.m45.ihep.su 2 + 0 xroot02.ncg.ingrid.pt 2 + 0 xrootd01-cmst1.pic.es 1 + 0 xrootd02-cmst1.pic.es 1 + 0 xrootd-cms-01.cr.cnaf.infn.it 1 + 0 xrootd-cms-02.cr.cnaf.infn.it 1 + 0 xrootd.cmsaf.mit.edu 2 + 0 xrootd-es-cie.ciemat.es 2 + 0 xrootd.hepgrid.uerj.br 2 + 0 xrootd.hep.kbfi.ee 2 + 0 xrootd-local.unl.edu 2 + 0 xrootd.rcac.purdue.edu 2 + 0 xrootd-redir.ultralight.org 2 + 1 xrootd-vanderbilt.sites.opensciencegrid.org 2 + 1 xroot.pp.rl.ac.uk 2 + diff --git a/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh new file mode 100755 index 0000000..4b882cc --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/cms_sam3_check.sh @@ -0,0 +1,939 @@ +#!/bin/bash +# +# Bockjoo Kim +# WHAT : Minimally large effort Emulation of CMS SAM Dashboard in plain cgi/HTML or for failed test alert +# Reference : https://monit-grafana.cern.ch/d/2LNN04NMk/user-carizapo-sitemon?orgId=11 +# https://monit-kibana-acc.cern.ch/kibana/app/kibana#/discover?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-17h,to:now))&_a=(columns:!(data.metric_name,data.service_flavour,data.dst_experiment_site,data.status),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:'monit_prod_sam3_enr_*',key:data.dst_experiment_site,negate:!f,params:(query:T2_US_Florida),type:phrase,value:T2_US_Florida),query:(match:(data.dst_experiment_site:(query:T2_US_Florida,type:phrase))))),index:'monit_prod_sam3_enr_*',interval:auto,query:(language:kuery,query:''),sort:!(metadata.timestamp,desc)) +# https://monit-kibana-acc.cern.ch/kibana/goto/7f572448ab2bbd36280ad87a3f499abe +# SAM test twiki : https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsSAMTests +# SAM test twiki : https://twiki.cern.ch/twiki/bin/view/CMS/SAMTestEmulationHowto +# Gitlab : https://gitlab.cern.ch/etf/cmssam/-/tree/master +# +#inputs=/opt/cms/services/HammerCloudXrootdMonitoring +inputs=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +notifytowhom=bockjoo__AT__gmail__dot__com +#THESITE=T2_US_Florida +#tld=rc.ufl.edu # change this +#port=8443 +#webroot=http://$(/bin/hostname -s).${tld}:${port}/cgi-bin +#thesite= +#used_percent_warning=70 # change this if needed +#used_percent=$(df -h /$(echo $inputs | cut -d/ -f2) | grep -v Filesystem | awk '{print $(NF-1)}' | sed 's#%##g') +#if [ $used_percent -gt $used_percent_warning ] ; then +# ( +# echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") +# echo "Subject: Warning not enough space in /opt" +# echo "Content-Type: text/html" +# echo "" +# echo "
"
+#     echo "$(/bin/hostname)" "$0" "
"
+#     echo "Used: $used_percent (%) vs Used Warning: $used_percent_warning (%)"
+#     du -sh $inputs/*
+#     echo "
" +# echo "" +# #cat $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +# echo +# ) | /usr/sbin/sendmail -t +#fi + +basename_0=$(basename $0) +gte="now-40m/m" +lte="now" +scroll="20m" +size=10000 +theprofile=CMS_CRITICAL_FULL +#theprofile=CMS_CRITICAL +#theprofile=CMS_FULL +DBID=9673 +DBID_CMSSST=9475 +EVERY_X_HOUR=1 +#unique_f=data.metric_name # data.CRAB_JobLogURL +##tie_breaker_id=metadata.kafka_timestamp # data.CRAB_TaskCreationDate +#tie_breaker_id=metadata.timestamp # data.CRAB_TaskCreationDate +unique_f=metadata.timestamp # data.CRAB_JobLogURL +tie_breaker_id=data.metric_name # data.CRAB_TaskCreationDate +tie_breaker_id=metadata.kafka_timestamp # +#index_json="{\"search_type\": \"query_then_fetch\", \"index\": [\"monit_prod_cms_rucio_raw_events*\"], \"ignore_unavailable\": true}" + +detail_link_prefix="https://monit-grafana.cern.ch/d/siYq3DxZz/wlcg-sitemon-test-details?orgId=20&var-id=" +cms_wiki_how_to_sam="https://twiki.cern.ch/twiki/bin/view/CMS/SAMTestEmulationHowto" + +#printf "Content-type: text/html\n\n" + +functions=$inputs/`basename ${0}`.fuctions.`date -u +%s` +perl -n -e 'print if /^####### BEGIN Functions 12345/ .. /^####### ENDIN Functions 12345/' < $0 | grep -v "Functions 12345" > $functions +source $functions +rm -f $functions + +#[ $# -gt 0 ] && thesite=$1 +#[ $# -gt 1 ] && gte=$2 +#[ $# -gt 2 ] && lte=$3 + + +#thestrings=`echo $QUERY_STRING | sed "s#&# #g"` +#for thestring in $thestrings ; do +# echo "$thestring" | grep -q -e "test_howto\|test_id\|test_output\|site\|gte\|lte\|match_all" +# [ $? -eq 0 ] || continue +# eval $thestring +# if [ "x$site" != "x" ] ; then +# thesite=$site +# fi +#done + +#thestrings=`echo $QUERY_STRING | sed "s#&# #g"` +#for thestring in $thestrings ; do +# echo "$thestring" | grep -q -e "test_howto\|test_id\|test_output\|site\|gte\|lte\|match_all" +# [ $? -eq 0 ] || continue +# if [ "x$test_id" != "x" ] ; then +# print_test_output $test_id $site +# exit 0 +# fi +# if [ "x$test_howto" != "x" ] ; then +# echo "
"
+#       echo "How to emulate this SAM test ( $test_howto )"
+#       thefunction=readme_$(echo $test_howto | sed "s|\.|__dot__|g" | sed "s|-|__dash__|g")
+#       $thefunction
+#       echo "
" +# exit 0 +# fi +#done + +#if [ "x$thesite" == "x" ] ; then +# thesite=$(basename $(readlink -f /cvmfs/cms.cern.ch/SITECONF/local)) +#fi +#if [ "x$thesite" == "x" ] ; then +# printf "Content-type: text/html\n\n" +# echo ERROR $(basename $0) provide the argument for the site name +# exit 0 +#fi +#if [ $(echo $thesite | wc -w) -ne 1 ] ; then +# printf "Content-type: text/html\n\n" +# echo ERROR $(basename $0) Unfortunately number of sitename is not unique or zero. +# exit 0 +#fi + + +#now_is=$(date +%s)_${thesite} +#now_is=$(date +%s)_american_sites +#echo "" > $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + +# All the template json files needed here found in $inputs +#sitemon_agg_metric_json=es.q.match_sitemon_raw_metric.json +token_txt=cms_sam3_check_token.txt +sort_search_json=cms_sam3_check_sort_search.json +sort_search_after_json=cms_sam3_check_sort_search_after.json +search_details=es.q.match_sitemon_raw_metric_search_details.json +sort_search_id_json=es.q.match_sitemon_raw_metric_sort_search_id.json +monit_prod_cmssst_search_json=monit_prod_cmssst_search.json + +[ -f $inputs/$token_txt ] || perl -n -e 'print if /^####### BEGIN token_txt/ .. /^####### ENDIN token_txt/' < $0 | grep -v "token_txt" > $inputs/$token_txt +perl -n -e 'print if /^####### BEGIN sort_search_json/ .. /^####### ENDIN sort_search_json/' < $0 | grep -v "sort_search_json" > $inputs/${sort_search_json}.in +perl -n -e 'print if /^####### BEGIN sort_search_after_json/ .. /^####### ENDIN sort_search_after_json/' < $0 | grep -v "sort_search_after_json" > $inputs/${sort_search_after_json}.in +perl -n -e 'print if /^####### BEGIN monit_prod_cmssst_search_json/ .. /^####### ENDIN monit_prod_cmssst_search_json/' < $0 | grep -v "monit_prod_cmssst_search_json" > $inputs/${monit_prod_cmssst_search_json}.in +startTIS=$(expr $(expr $(date +%s ) / 86400 - 5 ) \* 86400) +limitTIS=$(expr $(expr $(date +%s) / 900 ) \* 900 + 450) + +GTE=$(echo $gte | sed 's#/# #' | awk '{print $1}') +LTE=$(echo $lte | sed 's#/# #' | awk '{print $1}') +#sites="T2_US_Caltech T2_US_Florida T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Wisconsin T2_BR_SPRACE T2_BR_UERJ" +#sites=T2_US_Florida +#sites="T2_US_Florida T2_US_Caltech T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Wisconsin T2_BR_SPRACE T2_BR_UERJ T1_US_FNAL T2_US_Vanderbilt T2_UK_London_Brunel T3_US_Colorado" # T2_KR_KISTI T2_FR_CCIN2P3 T0_CH_CERN T1_UK_RAL" +#sites="T2_US_Florida T2_RU_ITEP" # T2_PK_NCP" # T2_US_Caltech T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Wisconsin T2_BR_SPRACE T2_BR_UERJ T1_US_FNAL T2_KR_KISTI T2_FR_CCIN2P3 T0_CH_CERN T1_UK_RAL" + +#sites="T2_US_Florida T2_AT_Vienna T2_BE_IIHE T2_BE_UCL T2_BR_SPRACE T2_BR_UERJ T2_CH_CERN T2_CH_CSCS T2_CN_Beijing T2_DE_DESY T2_DE_RWTH T2_EE_Estonia T2_ES_CIEMAT T2_ES_IFCA T2_FI_HIP T2_FR_GRIF_IRFU T2_FR_IPHC T2_GR_Ioannina T2_HU_Budapest T2_IN_TIFR T2_IT_Bari T2_IT_Legnaro T2_IT_Pisa T2_IT_Rome T2_KR_KISTI T2_PK_NCP T2_PL_Swierk T2_PT_NCG_Lisbon T2_RU_IHEP T2_RU_INR T2_RU_ITEP T2_RU_JINR T2_TR_METU T2_TW_NCHC T2_UA_KIPT T2_UK_London_Brunel T2_UK_London_IC T2_UK_SGrid_RALPP T2_US_Caltech T2_US_MIT T2_US_Nebraska T2_US_Purdue T2_US_UCSD T2_US_Vanderbilt T2_US_Wisconsin T0_CH_CERN" +if [ $# -lt 1 ] ; then + echo ERROR: $(basename $0) site + exit 1 +fi + +[ $# -gt 0 ] && thesite="$1" + +sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" -e "s#@@startTIS@@#$startTIS#" -e "s#@@limitTIS@@#$limitTIS#" $inputs/${monit_prod_cmssst_search_json}.in > $inputs/${monit_prod_cmssst_search_json} +curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/$token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID_CMSSST}/_msearch --data-binary "@$inputs/${monit_prod_cmssst_search_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##")_monit_prod_cmssst_search.out +#echo INFO check $inputs/$(basename $0 | sed "s#\.sh##")_monit_prod_cmssst_search.out +#exit 0 + +error_message="" +#error_message_html="" +#extra_metric_names="" +#for thesite in $sites ; do + + +#sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" $inputs/${sitemon_agg_metric_json}.in > ${sitemon_agg_metric_json}_${now_is} +#nhits=$(curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${sitemon_agg_metric_json}_${now_is}" 2>/dev/null | sed 's#"hits":#\nhits_total #' | grep ^hits_total | cut -d: -f2 | cut -d, -f1) + +#sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_json}.in > ${sort_search_json}_${now_is} +#nsearch=$(expr $nhits / $size) + +#echo "
"
+#echo "  Site:$thesite SAM Test Status  "
+#echo "<center><FONT color='green'> <b> Site:$thesite SAM Test Status </b> </FONT></center>"
+#echo "
  Site:$thesite SAM Test Status  
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html
+#echo INFO Site=$thesite nhits=$nhits query json follows
+#cat ${sort_search_json}_${now_is}
+#echo "
" + +#for i in $(seq 0 $nsearch) ; do +i=0 +while : ; do + i=$(expr $i + 1) + if [ $i -eq 1 ] ; then + sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_json}.in > $inputs/${sort_search_json} + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/$token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out + rm -f $inputs/${sort_search_json} + else + sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s|@@search_after@@|$search_after|" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_after_json}.in > $inputs/${sort_search_after_json} + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/$token_txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_after_json}" 2>/dev/null 1> $inputs/$(basename $0 | sed "s#\.sh##").$i.out + rm -f $inputs/${sort_search_after_json} + fi + #echo DEBUG $inputs/$(basename $0 | sed "s#\.sh##").$i.out_${now_is} + entry=$(sed "s#\"_source\":#\n\"_source\":#g" $inputs/$(echo $basename_0 | sed "s#\.sh##").$i.out | grep ^\"_source\": | wc -l) + search_after=$(sed 's#sort":#\nsort":#g' $inputs/$(basename $0 | sed "s#\.sh##").$i.out | grep ^sort | tail -1 | cut -d\[ -f2- | cut -d\] -f1) + #echo DEBUG entry=$entry + #echo DEBUG search_after=$search_after + [ $entry -lt $size ] && break + [ "x$search_after" == "xnull,null" ] && break + [ "x$search_after" == "x" ] && break +done + +#exit 0 + +# if [ $i -eq 0 ] ; then +# curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${sort_search_json}_${now_is}" 2>/dev/null 1> $(basename $0 | sed "s#\.sh##").$i.out_${now_is} +# search_after=$(sed 's#sort":#\nsort":#g' $(basename $0 | sed "s#\.sh##").$i.out_${now_is} | grep ^sort | tail -1 | cut -d\[ -f2- | cut -d\] -f1) +# else +# sed -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s|@@search_after@@|$search_after|" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_after_json}.in > ${sort_search_after_json}_${now_is} +# curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${sort_search_after_json}_${now_is}" 2>/dev/null 1> $(basename $0 | sed "s#\.sh##").$i.out_${now_is} +# search_after=$(sed 's#sort":#\nsort":#g' $(basename $0 | sed "s#\.sh##").$i.out_${now_is} | grep ^sort | tail -1 | cut -d\[ -f2- | cut -d\] -f1) +# fi +#done + +#ls $inputs/$(basename $0 | sed "s#\.sh##").*.out_${now_is} +#exit 0 + +#timestamps=$(sed 's#"data":#\n"data":#g' $inputs/$(basename $0 | sed "s#\.sh##").*.out | sed 's#,"#,\n"#g' | grep "^\"_id\"\|^\"metric_name\"\|^\"dst_hostname\"\|^\"status\"\|^\"service_flavour\"\|^\"timestamp\"" | grep "\"timestamp\"" | cut -d: -f2 | cut -d, -f1 | cut -d\} -f1 | sort -rn | uniq) +#LEGEND=" +#01:org.cms.WN-squid:(/cms/Role_lcgadmin) +#02:org.cms.WN-frontier:(/cms/Role_lcgadmin) +#03:org.cms.WN-basic:(/cms/Role_lcgadmin) +#04:org.cms.SRM-GetPFNFromTFC:(/cms/Role_production) +#05:org.cms.WN-xrootd-access:(/cms/Role_lcgadmin) +#06:org.cms.SE-xrootd-connection +#07:org.cms.WN-xrootd-fallback:(/cms/Role_lcgadmin) +#08:org.cms.WN-mc:(/cms/Role_lcgadmin) +#09:org.cms.WN-env:(/cms/Role_lcgadmin) +#10:org.cms.WN-isolation:(/cms/Role_lcgadmin) +#11:org.cms.SRM-VOPut:(/cms/Role_production) +#12:org.sam.CONDOR-JobSubmit:(/cms/Role_lcgadmin) +#13:org.cms.WN-analysis:(/cms/Role_lcgadmin) +#14:org.cms.SE-xrootd-contain +#15:org.cms.SRM-VOGet:(/cms/Role_production) +#16:org.cms.SE-xrootd-version +#17:org.cms.SE-xrootd-read +#99:org.cms.WN-cvmfs:(/cms/Role_lcgadmin)" +LEGEND=" +01:org.cms.WN-squid:(/cms/Role_lcgadmin) +02:org.cms.WN-frontier:(/cms/Role_lcgadmin) +03:org.cms.WN-basic:(/cms/Role_lcgadmin) +04:org.cms.SRM-GetPFNFromTFC:(/cms/Role_production) +05:org.cms.WN-xrootd-access:(/cms/Role_lcgadmin) +06:org.cms.SE-xrootd-connection +07:org.cms.WN-xrootd-fallback:(/cms/Role_lcgadmin) +08:org.cms.WN-mc:(/cms/Role_lcgadmin) +09:org.cms.WN-env:(/cms/Role_lcgadmin) +10:org.cms.WN-isolation:(/cms/Role_lcgadmin) +11:org.cms.SRM-VOPut:(/cms/Role_production) +12:org.sam.CONDOR-JobSubmit:(/cms/Role_lcgadmin) +13:org.cms.WN-analysis:(/cms/Role_lcgadmin) +14:org.cms.SE-xrootd-contain +15:org.cms.SRM-VOGet:(/cms/Role_production) +16:org.cms.SE-xrootd-version +17:org.cms.SE-xrootd-read +18:org.cms.SE-WebDAV-1connection +19:org.cms.SE-WebDAV-2ssl +20:org.cms.SE-WebDAV-3extension +21:org.cms.SE-WebDAV-4crt-read +22:org.cms.SE-WebDAV-5open-access +23:org.cms.SE-WebDAV-6crt-write +24:org.cms.SE-WebDAV-7macaroon +25:org.cms.SE-WebDAV-9summary +26:org.cms.CONDOR-Ping:(/cms-ce-token) +99:org.cms.WN-cvmfs:(/cms/Role_lcgadmin)" +metrics=$(printf "$LEGEND\n" | cut -d: -f2 | grep ^org | sed "s#^org#\\\\\\\|org#g") +metrics=$(echo $metrics | sed "s#\\\\|org#org#" | sed "s# ##g" | sed "s#\\\org#org#") +metrics=$(printf "$LEGEND\n" | cut -d: -f2 | grep ^org | sed "s#^org#\\\|org#g") +metrics=$(echo $metrics | sed "s#\\\|org#org#" | sed "s# ##g") + +output=$(sed 's#"data":#\n"data":#g' $inputs/$(basename $0 | sed "s#\.sh##").*.out | sed 's#,"#,\n"#g' | grep "^\"_id\"\|^\"metric_name\"\|^\"dst_hostname\"\|^\"status\"\|^\"service_flavour\"\|^\"timestamp\"" | grep -B 5 -A 5 "$metrics") +#exit 0 +if [ ] ; then + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + + for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do + [ "$sf" == "SRM" ] && continue + [ "$sf" == "XROOTD" ] && continue + [ "$sf" == "WEBDAV" ] && continue + nt=0 + for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + nt=$(for t in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do echo $t ; done | wc -l) + [ $nt -lt 10 ] && { nt=0 ; break ; } ; + done + done + [ $nt -lt 10 ] && { echo "
" Warning one or more CE has number of tests less than 10 "
" : nt=$nt >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html ; } ; + + if [ "x$thesite" == "x$THESITE" ] ; then + if [ $nt -lt 10 ] ; then + error_message="$error_message\nERROR one or more CE has number of tests less than 10 nt=$nt \n" + #else + # error_message="$error_message\nINFO All CEs have at least number of tests: nt=$nt \n" + error_message="" + fi + fi + + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +fi # if [ ] ; then + +i=0 +old_sf= +profile_history_site_link="https://monit-grafana.cern.ch/d/000000619/wlcg-sitemon-historical-profiles?orgId=20&var-vo=cms&var-dst_country=All&var-dst_federation=All&var-dst_tier=2&var-dst_experiment_site=${thesite}&var-service_flavour=All&var-profile=CMS_CRITICAL_FULL&var-dst_hostname=All&var-es_dst_hostname=All&var-recomputation=status" +#for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do +# echo $sf +#done +#exit 0 +for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do + i=$(expr $i + 1) + j=0 + [ "$sf" != "WEBDAV" ] && [ "$sf" != "XROOTD" ] && continue + #printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -m 1 -A 5 $t | grep "\"status\"" | cut -d\" -f4 + maxt=0 ; maxh= + #for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + #echo DEBUG $sf $h + #nt=$(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u | wc -l) + nt=$(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep "\"metric_name\"" | cut -d\" -f4 | sort -u | wc -l) + #for t in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + for t in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + m_name=$(echo $t | sed 's#-/# #g' | awk '{print $1}') + L=$(echo $(printf "$LEGEND\n" | grep $m_name | cut -d: -f1)) + if [ "x$L" == "x" ] ; then + echo "$extra_metric_names" | grep -q $t || extra_metric_names="$extra_metric_names
$t" + fi + done + [ $nt -gt $maxt ] && { maxt=$nt ; maxh=$h ; } ; + done +#done +#echo DEBUG maxt=$maxt maxh=$maxh +#exit 0 + lstring= + #for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + if [ "$h" == "$maxh" ] ; then + #[ $i -gt 1 ] && echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + #for t in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + for t in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep "\"metric_name\"" | cut -d\" -f4 | sort -u) ; do + m_name=$(echo $t | sed 's#-/# #g' | awk '{print $1}') + L=$(echo $(printf "$LEGEND\n" | grep $m_name | cut -d: -f1)) + [ "x$L" == "x" ] || { + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + twiki_toc=${L}_$(echo $m_name | sed "s#\.#_#g" | sed "s#-#_#g") + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + lstring="${lstring}|$L" + } ; + done + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + done + #echo DEBUG lstring=$lstring +#done +#exit 0 + #for h in $(printf "$output\n" | grep -A 2 -B 3 $sf | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do + profile_history_host_link="https://monit-grafana.cern.ch/d/000000619/wlcg-sitemon-historical-profiles?orgId=20&var-vo=cms&var-dst_country=All&var-dst_federation=All&var-dst_tier=2&var-dst_experiment_site=${thesite}&var-service_flavour=All&var-profile=CMS_CRITICAL_FULL&var-dst_hostname=${h}&var-es_dst_hostname=${h}&var-recomputation=status" + j=$(expr $j + 1) + if [ ] ; then + if [ $i -eq 1 ] ; then # First Service Flavor + if [ $j -eq 1 ] ; then + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + else + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + else + if [ $j -eq 1 ] ; then + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + else + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + fi + fi # if [ ] ; then + im=0 + for legend in $(echo $lstring | sed "s#|# #g") ; do + im=$(expr $im + 1) + t=$(printf "$LEGEND\n" | grep "${legend}:" | cut -d: -f2) + #status=$(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep -m 1 -A 5 $t | grep "\"status\"" | cut -d\" -f4) + #id=$(echo $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep -m 1 -B 5 $t | grep "\"_id\"" | cut -d\" -f4)) + #timestamp=$(echo $(printf "$output\n" | grep -A 2 -B 3 $sf | grep -A 4 -B 1 $h | grep -B 5 -A 5 $t | grep "\"timestamp\"" | cut -d\" -f3 | cut -d: -f2 | cut -d, -f1 | head -1)) # | cut -d: -f2 | cut -d, -f1 | head -1)) # | grep -m 1 -B 5 -A 5 $t | cut -d\" -f4)) + status=$(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -m 1 -A 5 $t | grep "\"status\"" | cut -d\" -f4) + echo $sf $h $legend t=$t status=$status + if [ "x$status" != "xOK" ] ; then + error_message="$error_message\n[$sf][$h][$t]" + fi + done # for legend in $(echo $lstring | sed "s#|# #g") ; do + done # for h in $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep "\"dst_hostname\"" | sort -u | cut -d\" -f4) ; do +done # for sf in $(printf "$output\n" | grep "\"service_flavour\"" | sort -u | cut -d\" -f4) ; do +if [ "x$error_message" != "x" ] ; then + #printf "$error_message\n" + #echo $thesite NOTOK + exit 1 +fi +exit 0 + +# done +# done +# done +#exit 0 + #id=$(echo $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -m 1 -B 5 $t | grep "\"_id\"" | cut -d\" -f4)) + #timestamp=$(echo $(printf "$output\n" | grep -A 2 -B 3 \"service_flavour\":\"$sf\" | grep -A 4 -B 1 \"dst_hostname\":\"$h\" | grep -B 5 -A 5 $t | grep "\"timestamp\"" | cut -d\" -f3 | cut -d: -f2 | cut -d, -f1 | head -1)) # | cut -d: -f2 | cut -d, -f1 | head -1)) # | grep -m 1 -B 5 -A 5 $t | cut -d\" -f4)) + #YmdHM=$(date -d@$(expr $timestamp / 1000) +%Y%m%d_%H:%M) + + #echo DEBUG sf=$sf h=$h legend=$legend t=$t + + #if [ "x$timestamp" == "x" ] ; then + # YmdHM=SPARSE + #else + # YmdHM=$(date -d@$(expr $timestamp / 1000) +%m%d%H%M) + #fi + bgcolor=red + [ "x$status" == "xOK" ] && bgcolor="#90EE90" # green + [ "x$status" == "xCRITICAL" ] && bgcolor=red + [ "x$status" == "xWARNING" ] && bgcolor=orange + [ "x$status" == "xUNKNOWN" ] && bgcolor=grey + [ "x$status" == "x" ] && bgcolor=white + m_name=$(echo $t | sed 's#-/# #g' | awk '{print $1}') + L=$(echo $(printf "$LEGEND\n" | grep $m_name | cut -d: -f1)) + #if [ "x$L" != "x" ] ; then + if [ "x$status" == "x" ] ; then + : + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + #status=NT + #detail_link=SPARSE + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + else + if [ "x$timestamp" == "x" ] ; then + status=SPARSE + detail_link=SPARSE + #else + #sed -e "s#@@gte@@#$gte#" -e "s#@@lte@@#$lte#" -e "s#@@id@@#$id#" -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" $inputs/${sort_search_id_json}.in > $inputs/${sort_search_id_json}_${now_is} + #eval $(curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@$inputs/${sort_search_id_json}_${now_is}" 2>/dev/null | python -m json.tool | grep "\"metric_name\"\|\"timestamp\"" | sed "s#:#=#g" | sed "s# ##g" | sed 's#"##g' | sed "s#,##g" | head -2 | sed "s#timestamp=#thetimestamp=#g" | sed "s#/#__slash__#g" | sed "s#=#__equals__#g" | sed "s#metric_name__equals__#metric_name=#" | sed "s#thetimestamp__equals__#thetimestamp=#") + + #detail_link="https://monit-grafana.cern.ch/d/siYq3DxZz/wlcg-sitemon-test-details?orgId=20&var-metric=${metric_name}&var-dst_hostname=${h}&var-timestamp=${thetimestamp}" + #detail_link=$(echo ${detail_link} | sed "s#__slash__#/#g" | sed "s#__equals__#=#g") # / is %2F = is %3D + fi + #echo "" + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + fi + if [ "x$status" != "xOK" ] ; then + #error_message="$error_message\n[$sf][$h][$t] = $status ${detail_link_prefix}${id}\n" + #error_message="$error_message\n[$sf][$h][$t] = $status ${webroot}/$(basename $0)?test_id=$id&site=$thesite\n${webroot}/$(basename $0)\n" + #if [ "x$thesite" == "x$THESITE" ] ; then + error_message="$error_message\n[$sf][$h][$t] = $status \n${detail_link}\n${webroot}/$(basename $0)?test_id=$id&site=$thesite\n\n_id=$id metric_name=${metric_name} thetimestamp=$thetimestamp\n${detail_link}\n${webroot}/$(basename $0)?test_id=$id&site=$thesite\n${webroot}/$(basename $0)\n\nContent of input json\n$([ -f $inputs/${sort_search_id_json}_${now_is} ] && cat $inputs/${sort_search_id_json}_${now_is})" + #fi + #error_message_html="$error_message_html
[$thesite][$sf][$h][$t] = $status
${detail_link}
" + fi + #rm -f $inputs/${sort_search_id_json}_${now_is} + done + #echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + done +done +if [ "x$error_message" != "x" ] ; then + printf "$error_message\n" +fi +exit 0 +#echo "
SitenameFlavor Host
    $L $L
$thesite $sf $h
   $h
 $sf $h
   $h $L NT
$YmdHM 		$status
T:$YmdHM 		$status
$YmdHM
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +#echo "
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +#done # for thesite in $sites + +echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +for l in $LEGEND ; do + L=$(echo $l | cut -d: -f1) + t=$(echo $l | cut -d: -f2) + echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + +done +echo "
LegendMetric Name
$L $t
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "Test numbers > 17 are non-critical CMS tests" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo " Extra Metric Names
" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo " $extra_metric_names " >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +echo "" >> $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + +#echo error_message=$error_message +if [ "x$error_message" != "x" ] ; then + #printf "$error_message\n" + #printf "$(/bin/hostname -s) $(basename $0) SAM3 Test Alert\n$error_message\n" | mail -s "ERROR SAM3 Test Alert" $(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + State=ERROR + nline_error_message=$(printf "$error_message\n" | grep "$THESITE\|All CEs have at least number of tests" | wc -l) + [ $nline_error_message -lt 3 ] && State=INFO +#echo DEBUG sending an email + +( + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Subject: State=$State SAM3 Test Alert" + echo "Content-Type: text/html" + echo " Start of Content
Number of Error Message Lines: $nline_error_message
" + printf "$error_message\n" | head -2 + sed -i 's###' $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + echo "" + echo $(/bin/hostname -s) $(basename $0) SAM3 Test Alert "
" + echo ERROR MESSAGE $error_message + echo "$error_message_html" + echo "
" + cat $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + echo +) | /usr/sbin/sendmail -t +else +#echo DEBUG sending an email +#if [ $(expr $(date +%H) % $EVERY_X_HOUR) -eq 0 -a $(date +%M) -lt 15 ] ; then +( + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Subject: OK SAM3 Tests" + echo "Content-Type: text/html" + echo + + cat $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html + echo +) # | /usr/sbin/sendmail -t +#fi +fi +rm -f $inputs/$(echo $basename_0 | sed "s#.sh##")_${now_is}.html +rm -f $inputs/${sitemon_agg_metric_json}_${now_is} +rm -f $inputs/${sort_search_json}_${now_is} +rm -f $inputs/${sort_search_after_json}_${now_is} +rm -f $inputs/$(basename $0 | sed "s#\.sh##").*.out_${now_is} + +exit 0 +####### BEGIN monit_prod_cmssst_search_json +{"search_type":"query_then_fetch","index":["monit_prod_cmssst_*"]} +{"query":{"bool":{"must":[{"match_phrase":{"metadata.type":"ssbmetric"}},{"match_phrase":{"metadata.type_prefix":"raw"}},{"match_phrase":{"metadata.path":"sts15min"}}],"filter":{"range":{"metadata.timestamp":{"gte":@@startTIS@@,"lt":@@limitTIS@@,"format":"epoch_second"}}}}},"_source":{"includes":["metadata.timestamp","metadata.kafka_timestamp","data.name","data.status"]},"size":4096,"sort":[{"metadata.timestamp":"desc"},{"metadata.kafka_timestamp":"desc"}]} +####### ENDIN monit_prod_cmssst_search_json + +####### BEGIN sort_search_json +{"search_type": "query_then_fetch", "index": ["monit_prod_sam3_enr_metric*"], "ignore_unavailable": true} +{"query": {"bool": {"filter": [{"range": {"metadata.timestamp": {"gte": "now-4h/h", "lte": "now", "format": "epoch_millis"}}}, {"query_string": {"analyze_wildcard": true, "query": "data.dst_experiment_site:@@thesite@@"}}]}}, "from": 0, "size": @@size@@, "sort": [{"@@unique_f@@": "desc"},{"@@tie_breakter_id@@": "asc"}]} +####### ENDIN sort_search_json +####### BEGIN sort_search_after_json +search_type": "query_then_fetch", "index": ["monit_prod_sam3_enr_metric*"], "ignore_unavailable": true} +{"query": {"bool": {"filter": [{"range": {"metadata.timestamp": {"gte": "@@gte@@", "lte": "@@lte@@", "format": "epoch_millis"}}}, {"query_string": {"analyze_wildcard": true, "query": "data.dst_experiment_site:@@thesite@@"}}]}}, "from": 0, "size": @@size@@, "search_after": [@@search_after@@], "sort": [{"@@unique_f@@": "desc"},{"@@tie_breakter_id@@": "asc"}]} +####### ENDIN sort_search_after_json +####### BEGIN token_txt + +####### ENDIN token_txt +####### BEGIN Functions 12345 +# Functions +function readme_org__dot__cms__dot__WN__dash__analysis () { + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-analysis.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__basic () { + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-basic + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__env () { + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-env + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__isolation () { #10 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-isolation + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__squid () { #1 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "[ -f \$SAME_SENSOR_HOME/tests/CE-cms-squid ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-squid \$SAME_SENSOR_HOME/tests" + echo "[ -f \$SAME_SENSOR_HOME/tests/test_frontier.py ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/test_frontier.py \$SAME_SENSOR_HOME/tests" + echo \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-squid.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__frontier () { #2 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "[ -f \$SAME_SENSOR_HOME/tests/CE-cms-frontier ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-frontier \$SAME_SENSOR_HOME/tests" + echo "[ -f \$SAME_SENSOR_HOME/tests/CMSSW_frontier.sh ] || cp -pR \$SAME_SENSOR_HOME/../FroNtier/tests/CMSSW_frontier.sh \$SAME_SENSOR_HOME/tests" + echo \$SAME_SENSOR_HOME/../FroNtier/tests/CE-cms-frontier.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__xrootd__dash__access () { #5 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-xrootd-access.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__xrootd__dash__fallback () { #7 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo \$SAME_SENSOR_HOME/tests/CE-cms-xrootd-fallback.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__cms__dot__WN__dash__mc () { #8 + echo "[0] Go to the WN where the test failed" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "[ -f \$SAME_SENSOR_HOME/tests/CE-cms-mc ] || cp -pR \$SAME_SENSOR_HOME/../MonteCarlo/tests/CE-cms-mc \$SAME_SENSOR_HOME/tests" + echo "[ -d \$SAME_SENSOR_HOME/cms-MC-test ] || cp -pR \$SAME_SENSOR_HOME/../MonteCarlo/cms-MC-test \$SAME_SENSOR_HOME/" + echo \$SAME_SENSOR_HOME/../MonteCarlo/tests/CE-cms-mc.sing + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " +} + +function readme_org__dot__sam__dot__CONDOR__dash__JobSubmit () { #12 + echo "CASE: htcondor" + echo "[0] Go to a condor-g submission machine" + echo "[1] Creat a job submission file, e.g., nagrun.sub with the following content" + cat $inputs/nagrun.sub + echo + echo "[2] Prepare the executable and inputs" + echo "cd" + echo "mkdir JobSubmit/samjob" + echo "cd JobSubmit/samjob" + echo "for f in nagrun.sub nagrun.sh gridjob.tgz wnlogs.tgz ; do" + echo " gfal-copy -r -p gsiftp://cmsio.rc.ufl.edu/cmsuf/t2/operations/samjob/$f ./" + echo "done" + echo + echo cd .. + echo source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/current/el7-x86_64/setup.sh + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo "condor_submit nagrun.sub" + echo "watch condor_q" + echo "echo Check samjob/gridjob.{out,err,log} and wnlogs.tgz" + echo "echo The content of samjob/gridjob.out should be same as the output from the actual SAM CONDOR-JobSubmit test" + + +} + +function readme_org__dot__cms__dot__SRM__dash__GetPFNFromTFC () { #4 + echo "[0] Go to a SAM machine with a grid UI" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "echo Download gridmon" + echo cd \$SAME_SENSOR_HOME/../SRMv2/tests + echo "[ -d python-GridMon ] || { echo Download gridmon ; git clone https://github.com/ARGOeu/python-GridMon.git ; } ;" + echo "[ -d gridmon ] || ln -s python-GridMon/gridmon" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests # to import simplejson and gridmon" + echo "[ -w /var/lib/gridprobes ] || sudo /bin/bash -c \"[ -d /var/lib/gridprobes ] || mkdir /var/lib/gridprobes ; chown \$(id -un):\$(id -gn) /var/lib/gridprobes\"" + + echo \$SAME_SENSOR_HOME/../SE/srmvometrics.py -H cmsio.rc.ufl.edu -m org.cms.SRM-GetPFNFromTFC + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SRM__dash__VOPut () { #11 + echo "[0] Go to a SAM machine with a grid UI" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "echo Download gridmon" + echo cd \$SAME_SENSOR_HOME/../SRMv2/tests + echo "[ -d python-GridMon ] || { echo Download gridmon ; git clone https://github.com/ARGOeu/python-GridMon.git ; } ;" + echo "[ -d gridmon ] || ln -s python-GridMon/gridmon" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests # to import simplejson and gridmon" + echo "[ -w /var/lib/gridprobes ] || sudo /bin/bash -c \"[ -d /var/lib/gridprobes ] || mkdir /var/lib/gridprobes ; chown \$(id -un):\$(id -gn) /var/lib/gridprobes ; [ -f /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ] || mkdir -p /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ; touch /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu/testFileIn.txt\"" + + echo \$SAME_SENSOR_HOME/../SE/srmvometrics.py -H cmsio.rc.ufl.edu -m org.cms.SRM-VOPut + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SRM__dash__VOGet () { #15 + echo "[0] Go to a SAM machine with a grid UI" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "echo Download gridmon" + echo cd \$SAME_SENSOR_HOME/../SRMv2/tests + echo "[ -d python-GridMon ] || { echo Download gridmon ; git clone https://github.com/ARGOeu/python-GridMon.git ; } ;" + echo "[ -d gridmon ] || ln -s python-GridMon/gridmon" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests # to import simplejson and gridmon" + echo "[ -w /var/lib/gridprobes ] || sudo /bin/bash -c \"[ -d /var/lib/gridprobes ] || mkdir /var/lib/gridprobes ; chown \$(id -un):\$(id -gn) /var/lib/gridprobes ; [ -f /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ] || mkdir -p /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu ; touch /var/lib/gridprobes/ops/org.cms/SRM/cmsio.rc.ufl.edu/testFileIn.txt\"" + + echo \$SAME_SENSOR_HOME/../SE/srmvometrics.py -H cmsio.rc.ufl.edu -m org.cms.SRM-VOGet + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__connection () { #6 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__contain () { #14 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__read () { #17 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function readme_org__dot__cms__dot__SE__dash__xrootd__dash__version () { #16 + echo "[0] Go to a SAM machine with a grid UI (an lxplus machine is better for this test)" + echo "ssh cmsuser@wn" + echo "" + echo "[1] Download SAM Test if not " + echo "cd" + echo "git clone https://gitlab.cern.ch/etf/cmssam.git" + echo "" + echo "[2] Setup and Run the test" + echo "vi \$X509_USER_PROXY # as needed copy \$X509_USER_PROXY from local grid proxy generating machine and paste its content here." + echo export X509_CERT_DIR=/cvmfs/cms.cern.ch/grid/etc/grid-security/certificates + echo export X509_USER_PROXY=\$X509_USER_PROXY # for example X509_USER_PROXY=/tmp/x509up_u11234567890 + echo export SAME_SENSOR_HOME=\$HOME/cmssam/SiteTests/testjob + echo "python -c 'from XRootD import client' 2>/dev/null || sudo /bin/bash -c \"yum install -y python-xrootd\"" + echo "export PYTHONPATH=\$PYTHONPATH:\$SAME_SENSOR_HOME/../SRMv2/tests/nap" + echo "cd \$SAME_SENSOR_HOME/../SRMv2/tests/nap # the test python requires nap from https://gitlab.cern.ch:8443/etf/nap" + echo \$SAME_SENSOR_HOME/../SE/cmssam_xrootd_endpnt.py -H cmsio5.rc.ufl.edu -P 1094 -S T2_US_Florida -4 -C /dev/null -d + echo "" + echo "[3] References" + echo "{1} SAM Test Scripts " + echo "{2} CompOpsSAMTests " + echo "{3} Packages List" +} + +function print_test_output () { + id=$1 + thesite=$2 + now_is=$(date +%s)_${thesite} + sed -e "s#@@thesite@@#$thesite#" -e "s#@@profile@@#$theprofile#" -e "s#@@size@@#$size#" -e "s#@@unique_f@@#$unique_f#" -e "s#@@tie_breakter_id@@#$tie_breaker_id#" -e "s|@@id@@|$id|" $inputs/${search_details}.in > ${search_details}_${now_is} + echo "" + curl -H "Content-Type: application/x-ndjson" -H "Authorization: Bearer $(cat $inputs/token.txt)" -XGET https://monit-grafana.cern.ch/api/datasources/proxy/${DBID}/_msearch --data-binary "@${search_details}_${now_is}" 2>/dev/null | sed 's#"data":#\n"data":#g' | sed 's#,"#,\n"#g' | grep ^\"details\" | cut -d\" -f4- | sed 's|",||' + echo "" + rm -f ${search_details}_${now_is} +} + +####### ENDIN Functions 12345 diff --git a/AAAOps/FedProbeSendAAAMetrics/compare_timeout_list.sh b/AAAOps/FedProbeSendAAAMetrics/compare_timeout_list.sh new file mode 100755 index 0000000..63a0433 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/compare_timeout_list.sh @@ -0,0 +1,8 @@ +#!/bin/bash +list1=$(grep ^T[0-9]_ create_fedmaps.log | awk '{print $1}' | sort -u) +list2=$(cat aaa_federation.log | sed "s#'siteName'#\nsiteName'#g" | grep ^siteName | awk '{print $2" "$13" "$15" "$17}' | grep timeout | cut -d\' -f2 | sort -u) + +echo "$(echo $list1)" +echo "$(echo $list2)" + +exit 0 diff --git a/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py new file mode 100755 index 0000000..7b84e0a --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/create_fedmaps.py @@ -0,0 +1,1463 @@ +try: import xml.etree.ElementTree as ET +except ImportError: from elementtree import ElementTree as ET +try: import json +except ImportError: import simplejson as json + +import sys +#import urllib2, httplib, sys +#import httplib, sys + +try : import urllib +except : import urllib2 + +try: + # For Python 3.0 and later + import urllib.request +except ImportError: + # Fall back to Python 2's urllib2 + import urllib2 + +# from XRDFED-kibana-probe.py +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +#import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import socket +#import path +#import os.path +from os import path +import numpy as np + +# global vars: sites, output +# cmsTopology : from vofeed, +# sites : from vofeed and updated as we proceed +# output : {"prod" : [...], "trans" : [...], "nowhere" : [...]} + +THEPATH='/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/' +#VOFEED='http://dashb-cms-vo-feed.cern.ch/dashboard/request.py/cmssitemapbdii' +VOFEED='https://cmssst.web.cern.ch/cmssst/vofeed/vofeed.xml' +LOCKFILE=THEPATH+'create_fedmaps.lock' +timeout_sec = 10 * 60 + +output = {"prod" : [], "trans" : [], "nowhere" : []} +sites = {} +xrdmapc_output_prod = '' +xrdmapc_output_prod_2 = '' +xrdmapc_output_tran = '' +gRedirectors=['cms-xrd-global01.cern.ch', 'cms-xrd-global02.cern.ch'] +tRedirectors=['cms-xrd-transit.cern.ch'] +topRedirectors = [gRedirectors[0],gRedirectors[1],]+tRedirectors + +if not path.exists(THEPATH+'mapHostSitenames.py') : + with open(THEPATH+'mapHostSitenames.py','w') as data : data.write('HostSitenames = {}') + +if not path.exists(THEPATH+'siteStoages.py') : + with open(THEPATH+'siteStoages.py','w') as data : data.write('SiteStorages = {}') + +if not path.exists(THEPATH+'DNSARecords.py') : + with open(THEPATH+'DNSARecords.py','w') as data : data.write('DNSARecords = {}') + +if not path.exists(THEPATH+'xrdVersions.py') : + with open(THEPATH+'xrdVersions.py','w') as data : data.write('XrdVersions = {}') + +from mapHostSitenames import HostSitenames +from siteStoages import SiteStorages +from DNSARecords import DNSARecords +from xrdVersions import XrdVersions + +class Alarm(Exception): + pass + +def alarm_handler(signum, frame): + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + clear_lock() + sys.exit(2) + raise Alarm + +def clear_lock(): + try: + os.unlink(LOCKFILE) + except (Exception,e) : + print ("could not remove lockfile:"+str(e)) + +def xrd_info(redirector,what): + theargs = [redirector, "query", "config", what] + if 'version' in what or 'role' in what : command = 'xrdfs' + else : + command = 'xrdmapc' + theargs = [ '--list', 'all', redirector ] + config_out = "unknown" + count = 0 + count_limit = 3 + timelimit = 180 # 2 + if 'xrdmapc' in command : + timelimit = 180 + count_limit = 1 + while (count < count_limit): + (errtext,out,err,elapsed) = run_xrd_commands(command, theargs, timelimit) #"xrdfs", + #[redirector, + # "query","config", # 1:kXR_QStats + # what]) # a_ll stats + + if 'xrdmapc' in command : + if out : + errtext = err + config_out = out + break + if err: + #errtext = '' + #config_out = 'error' + config_out = err.replace(b'\n',b'').decode() + else: + #config_out=out + #if 'xrdmapc' in command : + # break + #else : + config_out = out.replace(b'\n',b'').decode() + # # to check if count matters config_out=config_out+'+'+str(count) + if out.replace(b'\n',b'') : break + time.sleep(1) + count += 1 + #print ( 'DEBUG count = ',count,' config_out ',config_out, ' redirector ',redirector) + if not 'xrdmapc' in command : + if not config_out : config_out = 'timeout' + if 'Auth failed' in config_out : config_out = 'Auth failed' + return (errtext,config_out,out) + +def run_xrd_commands(cmd,args,timelimit): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + #xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + # "-DIConnectTimeout","30", + # "-DITransactionTimeout","60", + # "-DIRequestTimeout","60" ] + args + xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + #xrd_args = [ 'perl','-e',"alarm 30 ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + ] + args + #if 'xrdmapc' in cmd : + # xrd_args = [ cmd, ] + args + try: + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + ret = proc.returncode + elapsed = (time.time() - start) + #print ( ' out ',out ) + #if 'xrdmapc' in cmd : return ('',out,err,elapsed) + err_redir_index = err.rfind(b'Received redirection to') + err_index3010 = err.rfind(b'(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind(b'(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + errtxt = '' + else: + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind(b'Last server error') + if err_index >= 0: + err_end_index=err.find(b"\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except (Exception,e) : + errtext = errtxt + "Exception: "+str(e) + out = 'Try did not work :O' + print(out) + dev_null.close() + + return (errtxt,out,err,elapsed) + + +def getDataFromURL(url, header = {}): + try: + # For Python 3.0 and later + request = urllib.request.Request(url, headers=header) + urlObj = urllib.request.urlopen(request) + data = urlObj.read() + except ImportError: + request = urllib2.Request(url, headers=header) + urlObj = urllib2.urlopen(request) + data = urlObj.read() + return data + +# Hard-coded one +def getGlobalRedirectors(): + + siteName='T0_CH_CERN' + #gRedirectors=['cms-xrd-global01.cern.ch', 'cms-xrd-global02.cern.ch'] + #tRedirectors=['cms-xrd-transit.cern.ch'] + ret = {} + #ret[siteName] = {} + ret['sites'] = [] + ret['hosts'] = [] + ret['flavors'] = [] + ret['endpoints'] = [] + ret['name'] = [] + ret['contact'] = [] + ret['xrootd_version'] = [] + ret['xrootd_role'] = [] + ret['xrootd_servers'] = [] + ret['xrootd_storage'] = [] + if SiteStorages[ siteName ] : xrootd_storage = SiteStorages[ siteName ] + else : + xrootd_storage = getStorageFromStorageJson ( siteName ) + updateSiteStorage ( siteName , xrootd_storage ) + ret['federation'] = [] + + # Production + for gRedir in gRedirectors: + ret['sites'].append(siteName) + ret['hosts'].append(gRedir) + ret['flavors'].append('XROOTD') + ret['endpoints'].append(gRedir+':1094') + ret['name'].append('CERN-PROD') + ret['contact'].append(siteName) + #(err_info,xrootd_version,dump_info) = xrd_info(gRedir+':1094',"version") + #(err_info,xrootd_role,dump_info) = xrd_info(gRedir+':1094',"role") + ret['xrootd_version'].append('') + ret['xrootd_role'].append('') + ret['xrootd_servers'].append('_^_') + ret['xrootd_storage'].append(xrootd_storage) + ret['federation'].append('prod') + + # Transitional + for tRedir in tRedirectors: + ret['sites'].append(siteName) + ret['hosts'].append(tRedir) + ret['flavors'].append('XROOTD') + ret['endpoints'].append(tRedir+':1094') + ret['name'].append('CERN-PROD') + ret['contact'].append(siteName) + #(err_info,xrootd_version,dump_info) = xrd_info(gRedir+':1094',"version") + #(err_info,xrootd_role,dump_info) = xrd_info(gRedir+':1094',"role") + ret['xrootd_version'].append('') + ret['xrootd_role'].append('') + ret['xrootd_servers'].append('_^_') + ret['xrootd_storage'].append(xrootd_storage) + ret['federation'].append('trans') + return ret + +def getSites(): + XML = getDataFromURL(VOFEED) + XML = ET.fromstring(XML) + #sites = XML.findall('atp_site') + sites = XML.findall('.//atp_site') + ret = {} + for site in sites: + + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + services = site.findall('.//service') + ret[siteName] = {} + ret[siteName]['sites'] = [] + ret[siteName]['hosts'] = [] + ret[siteName]['flavors'] = [] + ret[siteName]['endpoints'] = [] + ret[siteName]['name'] = [] + ret[siteName]['contact'] = [] + ret[siteName]['xrootd_version'] = [] + ret[siteName]['xrootd_role'] = [] + ret[siteName]['xrootd_servers'] = [] + ret[siteName]['xrootd_storage'] = [] + ret[siteName]['federation'] = [] + #print ('DEBUG getSites() ret keys ',ret.keys() ) + #print ('getSites() ret["T0_CH_CERN"] ',ret["T0_CH_CERN"]) + name = '' + contact = '' + for site in sites: + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + #print ('DEBUG getSites() site ',siteName) + services = site.findall('.//service') + #ret[siteName]['name'].append(site.attrib['name']) + if site.attrib['name'] : name = site.attrib['name'] + contact = None + try: + #ret[siteName]['contact'].append(site.attrib['contact']) + contact = site.attrib['contact'] + except: + #ret[siteName]['contact'].append("") + contact = siteName + #if site.attrib['contact'] : contact = site.attrib['contact'] + + #print ('DEBUG getSites() site contact ',) + for service in services: + flavor = service.attrib['flavour'] + try: + endpoint=service.attrib['endpoint'] + except: + endpoint=service.attrib['flavour'] + + xrootd_version='__' + xrootd_role='__' + #if flavor in 'XROOTD': + # (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + # (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + serviceName = service.attrib['hostname'] + + #print ('DEBUG getSites() services siteName ',siteName) + ret[siteName]['sites'].append(siteName) + ret[siteName]['hosts'].append(serviceName) + ret[siteName]['flavors'].append(flavor) + ret[siteName]['endpoints'].append(endpoint) + ret[siteName]['xrootd_version'].append(xrootd_version) + ret[siteName]['xrootd_role'].append(xrootd_role) + ret[siteName]['xrootd_servers'].append('_^_') + #print ('DEBUG getSites() getStorageFromStorageJson siteName ',siteName) + xrootd_storage = getStorageFromStorageJson ( siteName) + #print ('DEBUG getSites() xrootd_storage ',xrootd_storage) + #if siteName in SiteStorages : print (' found it ') + #else : print ('not found ') + if siteName in SiteStorages : + #print ( 'A ') + xrootd_storage = SiteStorages[ siteName ] + else : + #print ( 'B ') + xrootd_storage = getStorageFromStorageJson ( siteName ) + updateSiteStorage ( siteName , xrootd_storage ) + #print ('DEBUG getSites() done sitestorage ') + ret[siteName]['xrootd_storage'].append(xrootd_storage) + ret[siteName]['name'].append(name) + ret[siteName]['contact'].append(contact) + ret[siteName]['federation'].append('nowhere') + #print ('DEBUG getSites() done ') + return ret + + +def getStorageFromStorageJson ( site ): + import os.path + from os import path + if not path.exists('/cvmfs/cms.cern.ch/SITECONF/'+site+'/storage.json') : return 'no_storage_json' + with open('/cvmfs/cms.cern.ch/SITECONF/'+site+'/storage.json') as f: storage = f.read() + try: + storage = json.loads(str(storage)) + except: + return 'storage_json_bad' + xrootd_volume = 'info_missing' + #print ('DEBUG getStorageFromStorageJson ( site ) before for vol in storage = ',storage) + #for line in xrdmapc_output_prod.splitlines() : + # line = line.decode() # python3 + #for vol in storage : + for index in range(len(storage)): + vol=storage[index] + volume = str(vol['volume']) # ' '.join([str(elem) for elem in str(vol['volume']).split('_')]) + #print ('DEBUG getStorageFromStorageJson ( site ) volume = ',volume) + if 'FEDERATION' in volume.upper() : continue + #print ('DEBUG getStorageFromStorageJson ( site ) FEDERATION not in volume ',volume, ' vol protocols len ',len(vol['protocols']),' vol ',vol['protocols']) + #for proto in vol['protocols'] : + for index_proto in range(len(vol['protocols'])) : + proto=vol['protocols'][index_proto] + #print ( 'DEBUG getStorageFromStorageJson ( site ) storage index ', index_proto, ' volume ',volume, ' type ',type(proto) ) + #print ( 'DEBUG getStorageFromStorageJson ( site ) storage index ', index_proto, ' volume ',volume, proto['protocol'] ) + try : + if 'XRootD' in str(proto['protocol']) : xrootd_volume = volume + except : + continue + + #print ('DEBUG getStorageFromStorageJson ( site ) after for vol in storage') + if 'info_missing' in xrootd_volume : + #for vol in storage : + for index in range(len(storage)): + vol=storage[index] + volume = str(vol['volume']) # ' '.join([str(elem) for elem in str(vol['volume']).split('_')]) + if 'FEDERATION' in volume.upper() : continue + #for proto in vol['protocols'] : + for index_proto in range(len(vol['protocols'])) : + proto=vol['protocols'][index_proto] + #print ( 'storage volume ',volume,proto['protocol'] ) + try : + if 'WebDAV' in str(proto['protocol']) : xrootd_volume = volume + except : + continue + #print ('DEBUG getStorageFromStorageJson ( site ) site volume ', site, xrootd_volume ) + # 3 ( + #print ( 'storage volume ', len(storage), (str(storage[0]['volume'])).split('_')[1:]) + #print ( 'storage volume ', len(storage), storage[1]) + #print ( 'storage volume ', len(storage), storage[2]) + return xrootd_volume + +def getStorages(): + XML = getDataFromURL(VOFEED) + XML = ET.fromstring(XML) + #sites = XML.findall('atp_site') + sites = XML.findall('.//atp_site') + ret = {} + for site in sites: + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + #services = site.findall('.//service') + ret[siteName] = {} + ret[siteName]['sites'] = [] + ret[siteName]['hosts'] = [] + ret[siteName]['flavors'] = [] + ret[siteName]['endpoints'] = [] + ret[siteName]['name'] = [] + ret[siteName]['contact'] = [] + ret[siteName]['xrootd_version'] = [] + ret[siteName]['xrootd_role'] = [] + ret[siteName]['xrootd_servers'] = [] + ret[siteName]['xrootd_storage'] = [] + name = '' + contact = '' + storage_guessed = '' + for site in sites: + groups = site.findall('.//group') + siteName = None + for i in groups: + if i.attrib['type'] == 'CMS_Site': + siteName = groups[1].attrib['name'] + break + if not siteName: + continue + se_resources = site.findall('.//se_resource') + storage_guessed = 'NI' + for i in se_resources : + if i.attrib['id'] == 'RD3PCP': + try : + path = se_resources[1].attrib['path'] + #print (siteName, path ) + if 'HADOOP' in path.upper() : storage_guessed = 'hadoop' + if 'DCACHE' in path.upper() : storage_guessed = 'dcache' + if '/PNFS/' in path.upper() : storage_guessed = 'dcache' + if 'CEPH' in path.upper() : storage_guessed = 'ceph' + if 'LUSTRE' in path.upper() : storage_guessed = 'lustre' + if 'STORM' in path.upper() : storage_guessed = 'posix' + if 'GPFS' in path.upper() : storage_guessed = 'gpfs' + if 'EOS' in path.upper() : storage_guessed = 'eos' + except : + storage_guessed = 'NI' + if i.attrib['id'] == 'WRDEL3PCP' : + try : + path = se_resources[1].attrib['path'] + #print (siteName, path ) + if 'HADOOP' in path.upper() : storage_guessed = 'hadoop' + if 'DCACHE' in path.upper() : storage_guessed = 'dcache' + if '/PNFS/' in path.upper() : storage_guessed = 'dcache' + if 'CEPH' in path.upper() : storage_guessed = 'ceph' + if 'LUSTRE' in path.upper() : storage_guessed = 'lustre' + if 'STORM' in path.upper() : storage_guessed = 'posix' + if 'GPFS' in path.upper() : storage_guessed = 'gpfs' + if 'EOS' in path.upper() : storage_guessed = 'eos' + except : + storage_guessed = 'NI' + + services = site.findall('.//service') + #ret[siteName]['name'].append(site.attrib['name']) + if site.attrib['name'] : name = site.attrib['name'] + try: + #ret[siteName]['contact'].append(site.attrib['contact']) + contact = site.attrib['contact'] + except: + #ret[siteName]['contact'].append("") + contact = siteName + #if site.attrib['contact'] : contact = site.attrib['contact'] + + for service in services: + flavor = service.attrib['flavour'] + try: + endpoint=service.attrib['endpoint'] + except: + endpoint=service.attrib['flavour'] + + xrootd_version='__' + xrootd_role='__' + #if flavor in 'XROOTD': + # (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + # (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + serviceName = service.attrib['hostname'] + if 'XROOTD' in flavor : + #print ( siteName, serviceName ) + if 'HADOOP' in serviceName.upper() : storage_guessed = 'hadoop' + if 'DCACHE' in serviceName.upper() : storage_guessed = 'dcache' + if '/PNFS/' in serviceName.upper() : storage_guessed = 'dcache' + if 'CEPH' in serviceName.upper() : + storage_guessed = 'ceph' + #print (siteName, serviceName, 'ceph') + if 'LUSTRE' in serviceName.upper() : storage_guessed = 'lustre' + if 'STORM' in serviceName.upper() : storage_guessed = 'posix' + if 'GPFS' in serviceName.upper() : storage_guessed = 'gpfs' + if 'EOS' in serviceName.upper() : storage_guessed = 'eos' + + ret[siteName]['sites'].append(siteName) + ret[siteName]['hosts'].append(serviceName) + ret[siteName]['flavors'].append(flavor) + ret[siteName]['endpoints'].append(endpoint) + ret[siteName]['xrootd_version'].append(xrootd_version) + ret[siteName]['xrootd_role'].append(xrootd_role) + ret[siteName]['xrootd_servers'].append('_^_') + ret[siteName]['xrootd_storage'].append( siteName+'+'+storage_guessed ) + #ret[siteName]['xrootd_storage_guessed'].append( storage_guessed ) + ret[siteName]['name'].append(name) + ret[siteName]['contact'].append(contact) + return ret + + +def parseHN(data): + parsedHNs = [] + for line in data.split('\n'): + if not len(line): continue + if ':' in line: line = line[:line.find(':')] + parsedHNs.append(line) + return parsedHNs + +def findDomain (h,idx) : + h=h.split('.')[::-1] + hlen=len(h) + #if idx > hlen - 1 : + # return '' + d='' + j=0 + #for i in h[idx:] : + for i in h[:idx] : + if j == 0 : d = i + else : d = d + '.' + i + j = j + 1 + return d + +def getIP(host_name): + try: + #host_name = socket.gethostname() + host_ip = socket.gethostbyname(host_name) + #print("Hostname : ",host_name) + #print("IP : ",host_ip) + return host_ip + except: + return "Unable.to.get.an.IP" + +def getIPReversed(host_name): + try: + ip=[ele for ele in reversed(getIP(host_name).split('.'))] + ipr='' + for p in ip : + if ipr : ipr=ipr+"."+p + else : ipr=ipr + p + return ipr + except: + return "Unable.to.get.a.Reverse.IP" + +def findSitenameOld(h): + mapHostSitename = [] + #print (' h ',h ) + ip='' + jlen = len(h.split('.')) + jd = h.split('.')[-2:] + jd.sort() + if 'in2p3.fr' in h : + #print ("findSitenames getIP for ",h) + ip=getIPReversed(h) + #print ( getIP(h), ip ) + #print ("findSitenames getIP for ",h, " is ",ip) + jlen = len(ip.split('.')) + jd = ip.split('.')[-2:] + jd.sort() + #print (' h ip jlen jd ',h,ip,jlen,jd) + + nmatch=0 + matched_site='' + matched_domain='' + for site in sites.keys() : + sited = sites[site]['hosts'][0].split(".")[-2:] + if 'in2p3.fr' in h : sited = getIPReversed ( sites[site]['hosts'][0] ).split(".")[-2:] + for idx in range ( len ( sites[site]['hosts'] ) ): + sited = sites[site]['hosts'][idx].split(".")[-2:] + sited.sort() + if 'in2p3.fr' in h : + sited = getIPReversed ( sites[site]['hosts'][idx] ).split(".")[-2:] + sited.sort() + #if site == 'T2_US_Vanderbilt' : + #print ( 'host ',sites[site]['hosts'][idx], ' jd ',jd, ' sited ',sited ) + if jd == sited : break + #if site == 'T2_US_Vanderbilt' : print (site, ' length ',len(sites[site]['flavors']), ' jd ',jd, ' sited ',sited) + if jd != sited : continue + if not "XROOTD" in set(sites[site]['flavors']) : continue + #socket.gethostbyname('sbgse1.in2p3.fr') + for i in range(len(sites[site]['flavors'])) : + for j in range(1, jlen): + adomain=findDomain(h,j) + bdomain=findDomain(sites[site]['hosts'][i],j) + if 'in2p3.fr' in h : + adomain=findDomain( ip , j) + bdomain=findDomain( getIPReversed ( sites[site]['hosts'][i] ), j) + + #print ( " j ", j, " adomain ",adomain, " bdomain ",bdomain) + if adomain == bdomain : + #print ( " returns j ", j, " adomain ",adomain, " bdomain ",bdomain) + #len(adomain.split('.')) + #if jlen - j > nmatch : + if len(adomain.split('.')) > nmatch : + matched_site=sites[site]['sites'][i] + nmatch = len(adomain.split('.')) # jlen - j + matched_domain=findDomain(adomain,nmatch) + #if jd == sited : print (h, site) + #if mapHostSitename : mapHostSitename[h] = [] + #print ( "appending site ",site) + mapHostSitename.append ( site ) + #mapHostSitename.append ( "m="+matched_site ) + #mapHostSitename.append ( matched_site ) + #if len( mapHostSitename ) > 1 and len ( matched_domain.split(".") ) < 3 : + # print ( h, matched_site, mapHostSitename) + return (matched_site, matched_domain) + +def findSitename(h): + global sites + gredirs = getGlobalRedirectors() + for idx in range ( len (gredirs['hosts']) ) : + if h in gredirs['hosts'][idx] : + return ( gredirs['sites'][idx], 'TOBEFIXED') + + mapHostSitename = [] + #print (' h ',h ) + ip='' + jlen = len(h.split('.')) + jd = h.split('.')[-2:] + jd.sort() + if 'in2p3.fr' in h : + #print ("findSitenames getIP for ",h) + ip=getIPReversed(h) + #print ( getIP(h), ip ) + #print ("findSitenames getIP for ",h, " is ",ip) + jlen = len(ip.split('.')) + jd = ip.split('.')[-2:] + jd.sort() + #print (' h ip jlen jd ',h,ip,jlen,jd) + + nmatch=0 + matched_site='' + matched_domain='' + for stype in [ 'xrootd','all' ] : + for site in sites.keys() : + sited = sites[site]['hosts'][0].split(".")[-2:] + if 'in2p3.fr' in h : sited = getIPReversed ( sites[site]['hosts'][0] ).split(".")[-2:] + for idx in range ( len ( sites[site]['hosts'] ) ): + sited = sites[site]['hosts'][idx].split(".")[-2:] + sited.sort() + if 'in2p3.fr' in h : + sited = getIPReversed ( sites[site]['hosts'][idx] ).split(".")[-2:] + sited.sort() + #if site == 'T2_US_Vanderbilt' : + #print ( 'host ',sites[site]['hosts'][idx], ' jd ',jd, ' sited ',sited ) + if jd == sited : break + #if site == 'T2_US_Vanderbilt' : print (site, ' length ',len(sites[site]['flavors']), ' jd ',jd, ' sited ',sited) + if jd != sited : continue + #print (" jd found ",jd ) + if stype == 'xrootd' : + if not "XROOTD" in set(sites[site]['flavors']) : continue + #socket.gethostbyname('sbgse1.in2p3.fr') + for i in range(len(sites[site]['flavors'])) : + for j in range(1, jlen): + adomain=findDomain(h,j) + bdomain=findDomain(sites[site]['hosts'][i],j) + if 'in2p3.fr' in h : + adomain=findDomain( ip , j) + bdomain=findDomain( getIPReversed ( sites[site]['hosts'][i] ), j) + + #print ( " j ", j, " adomain ",adomain, " bdomain ",bdomain) + if adomain == bdomain : + #print ( " returns j ", j, " adomain ",adomain, " bdomain ",bdomain) + #len(adomain.split('.')) + #if jlen - j > nmatch : + if len(adomain.split('.')) > nmatch : + matched_site=sites[site]['sites'][i] + nmatch = len(adomain.split('.')) # jlen - j + matched_domain=findDomain(adomain,nmatch) + #if jd == sited : print (h, site) + #if mapHostSitename : mapHostSitename[h] = [] + #print ( "appending site ",site) + #mapHostSitename.append ( site ) + #mapHostSitename.append ( "m="+matched_site ) + #mapHostSitename.append ( matched_site ) + if matched_site : break + + #if len( mapHostSitename ) > 1 and len ( matched_domain.split(".") ) < 3 : + # print ( h, matched_site, mapHostSitename) + return (matched_site, matched_domain) + +def updateHostSitename ( h, site ) : + #HostSitenames[h] = {} + HostSitenames[h] = site + with open(THEPATH+'mapHostSitenames.py','w') as data: + data.write('HostSitenames = ' + str(HostSitenames)) + +def updateSiteStorage ( site , storage ) : + #SiteStorages[ site ] = {} + SiteStorages[ site ] = storage + with open(THEPATH+'siteStoages.py','w') as data: + data.write('SiteStorages = ' + str(SiteStorages)) + + +def getRegionalRedirectors (): + global xrdmapc_output_prod + global xrdmapc_output_prod_2 + + #Lines=[] + #with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() + regionalRedirectors = {} + count = 0 + #for line in set(Lines) : + for line in xrdmapc_output_prod.splitlines() : + if type (line) is not str : line = line.decode() # python3 + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #if 'Man' in entries[1] : + if entries[1] == 'Man' : + #print ( "found Man in ",entries[1] ) + if int (entries[0]) == 1 : + endpoint = entries[2] + host = endpoint.split(':')[0] + if host in HostSitenames : + site=HostSitenames[host] + else : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + regionalRedirectors[site] = {} + regionalRedirectors[site]['endpoint'] = endpoint + regionalRedirectors[site]['host'] = host + regionalRedirectors[site]['flavors'] = 'XROOTD' + regionalRedirectors[site]['federation'] = 'prod' + + #count += 1 + #print ( regionalRedirectors ) + return regionalRedirectors + +def getRegionalRedirectorsFromFile (): + Lines=[] + with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() + regionalRedirectors = {} + #count = 0 + for line in set(Lines) : + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + if entries[1] == 'Man' : + if int (entries[0]) == 1 : + endpoint = entries[2] + host = endpoint.split(':')[0] + if host in HostSitenames : + site=HostSitenames[host] + else : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + regionalRedirectors[site] = {} + regionalRedirectors[site]['endpoint'] = endpoint + regionalRedirectors[site]['host'] = host + regionalRedirectors[site]['flavors'] = 'XROOTD' + regionalRedirectors[site]['federation'] = 'prod' + + #count += 1 + #print ( regionalRedirectors ) + return regionalRedirectors + + +def getXrootdServers () : + global xrdmapc_output_prod + global xrdmapc_output_prod_2 + global sites + #Lines=[] + #print ("DEBUG sites ",len ( sites )) + xrdServers = {} + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + count = 0 + for line in xrdmapc_output_prod.splitlines() : + #print("Line{}: {}".format(count, line.split())) + if type (line) is not str : line = line.decode() + entries=line.split() + #print ( entries ) + #if entries[0] == 'Srv' : + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + if '[' in endpoint : continue + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if host in HostSitenames : + site=HostSitenames[host] + else : + if 'localhost' in host : continue # give up + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + if site not in xrdServers : continue # skip if site not in vomsfeed + #print ( " host and site ",host, site ) + #print ( "DEBUG xrdServers[site]['endpoints'] ", xrdServers[site]['endpoints'] ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # site = 'T2_IN_TIFR' + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + # xrdServers[site]['hosts'].append ( 'se01.indiacms.res.in' ) + # xrdServers[site]['federation'].append ( 'prod' ) + #print ( endpoint, host, site ) + #if 'unl.edu' in endpoint : print ( endpoint, host , site, domain) + # addhoc + #if not site : + # #if 'vanderbilt.edu' in host : site = 'T2_US_Vanderbilt' + + #xrdServers[site] = {} + #print ( " site ",site) + #if site == 'T2_US_Nebraska' : print ( 'T2_US_Nebraska ', endpoint ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + #else : + #if 'localhost' in site: + # print (" site has localhost") + # xrdServers[site]={} #'endpoints':'','hosts':'','flavors':'','federation':''} + # xrdServers[site]['endpoints']=['localhost'] + # xrdServers[site]['hosts']=[host] + # xrdServers[site]['flavors']=['XROOTD'] + # xrdServers[site]['federation']=['prod'] + #else: + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'prod' ) + #print("Line{}: {}".format(count, line.split()), ' idx = ',idx, endpoint,site,xrdServers[site]['federation']) + + #count += 1 + #print ("Returning xrdServers") + return xrdServers + +def getXrootdServersFromFile () : + global sites + Lines=[] + with open(THEPATH+'out/xrdmapc_all_0.txt') as f: Lines = f.readlines() + print ("DEBUG sites ",len ( sites )) + xrdServers = {} + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + count = 0 + for line in Lines : + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #print ( entries ) + #if entries[0] == 'Srv' : + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + if '[' in endpoint : continue + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if host in HostSitenames : + site=HostSitenames[host] + else : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + #print ( host, site ) + #print ( xrdServers[site]['endpoints'] ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # site = 'T2_IN_TIFR' + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + # xrdServers[site]['hosts'].append ( 'se01.indiacms.res.in' ) + # xrdServers[site]['federation'].append ( 'prod' ) + #print ( endpoint, host, site ) + #if 'unl.edu' in endpoint : print ( endpoint, host , site, domain) + # addhoc + #if not site : + # #if 'vanderbilt.edu' in host : site = 'T2_US_Vanderbilt' + + #xrdServers[site] = {} + #print ( " site ",site) + #if site == 'T2_US_Nebraska' : print ( 'T2_US_Nebraska ', endpoint ) + #if '[::ffff:144.16.111.9]:11001' in endpoint : + # xrdServers[site]['endpoints'].append('se01.indiacms.res.in:11001') + #else : + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'prod' ) + #print("Line{}: {}".format(count, line.split()), ' idx = ',idx, endpoint,site,xrdServers[site]['federation']) + + #count += 1 + return xrdServers + +def getTransitionalXrootds () : + global sites + global xrdmapc_output_tran + #Lines=[] + #with open(THEPATH+'out/xrdmapc_trans_3.txt') as f: Lines = f.readlines() + xrdServers = {} + #print ("DEBUG sites ",sites.keys()) + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + count = 0 + for line in xrdmapc_output_tran.splitlines() : + if type (line) is not str : line = line.decode() + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #print ( entries + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if '[' in endpoint: continue + + if host in HostSitenames : + #print (' if host = ',host, ' HostSitenames ',HostSitenames ) + site=HostSitenames[host] + if not site : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + else : + #print (' else host = ',host ) + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + #print (' else host = ',host, ' site ',site ) + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'trans' ) + #print (' endpoint ',endpoint ) + + count += 1 + return xrdServers + +def getXrootdsFromTransFile () : + global sites + Lines=[] + with open(THEPATH+'out/xrdmapc_trans_3.txt') as f: Lines = f.readlines() + xrdServers = {} + #print ("DEBUG sites ",sites.keys()) + for site in sites : + #print ( ' site ',site ) + xrdServers[site] = {} + xrdServers[site]['endpoints'] = [] + xrdServers[site]['hosts'] = [] + xrdServers[site]['federation'] = [] + xrdServers[site]['flavors'] = [] + #return xrdServers + #count = 0 + for line in Lines : + #print("Line{}: {}".format(count, line.split())) + entries=line.split() + #print ( entries + idx = 0 + if entries[1] == 'Man' : idx = 2 + if entries[0] == 'Srv' : idx = 1 + if idx == 1 or idx == 2 : + endpoint = entries[idx] + host = endpoint.split(':')[0] + #(site,domain) = findSitename(host) + #from mapHostSitenames import HostSitenames + if '[' in endpoint: continue + + if host in HostSitenames : + #print (' if host = ',host, ' HostSitenames ',HostSitenames ) + site=HostSitenames[host] + if not site : + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + else : + #print (' else host = ',host ) + (site,domain) = findSitename(host) + updateHostSitename ( host, site ) + #print (' else host = ',host, ' site ',site ) + if not endpoint in xrdServers[site]['endpoints'] : + xrdServers[site]['endpoints'].append ( endpoint ) + xrdServers[site]['hosts'].append ( host ) + xrdServers[site]['flavors'].append ( 'XROOTD' ) + xrdServers[site]['federation'].append ( 'trans' ) + #print (' endpoint ',endpoint ) + + #count += 1 + return xrdServers + +def queryXrdmapc (redirector) : + global sites + global xrdmapc_output_prod + global xrdmapc_output_prod_2 + global xrdmapc_output_tran + if 'trans' in redirector : + (err_info,xrdmapc_output_tran,dump_info) = xrd_info(redirector,'') + with open(THEPATH+'out/xrdmapc_tran.txt','w' ) as f: + for line in xrdmapc_output_tran.splitlines() : + #f.write(str(line.decode())) + if type (line) is str : + #print ( "1 str "+line ) + f.write(line) + else : + #print ( "1 no str "+str(line.decode()) ) + f.write(str(line.decode())) + f.write('\n') + elif 'global02' in redirector : + (err_info,xrdmapc_output_prod_2,dump_info) = xrd_info(redirector,'') + with open(THEPATH+'out/xrdmapc_prod_2.txt','w' ) as f: + for line in xrdmapc_output_prod_2.splitlines() : + if type (line) is str : + #print ( "2 str "+line ) + f.write(line) + else : + #print ( "2 no str "+str(line.decode()) ) + f.write(str(line.decode())) + f.write('\n') + else : + (err_info,xrdmapc_output_prod,dump_info) = xrd_info(redirector,'') + with open(THEPATH+'out/xrdmapc_prod_1.txt','w' ) as f: + for line in xrdmapc_output_prod.splitlines() : + #f.write(str(line.decode())) + if type (line) is str : + #print ( "3 str "+line ) + f.write(line) + else : + #print ( "3 no str "+str(line.decode()) ) + f.write(str(line.decode())) + f.write('\n') + + +def updateXrdInfo (thesite) : + global sites + xrdInfo = sites[thesite] + xrd_idxes = findXROOTDIdxes ( xrdInfo ) + #print ( thesite, sites[thesite] ) + for idx in range(len(xrd_idxes)) : + endpoint = sites[thesite]['endpoints'][xrd_idxes[idx]] + (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + #xrdInfo['xrootd_version'][xrd_idxes[idx]] = xrootd_version + sites[thesite]['xrootd_version'][xrd_idxes[idx]] = xrootd_version + (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + #xrdInfo['xrootd_role'][xrd_idxes[idx]] = xrootd_role + sites[thesite]['xrootd_role'][xrd_idxes[idx]] = xrootd_role + #print ("DEBUG updateXrdInfo ",thesite, sites[thesite]['endpoints'], " xrootd_role ",xrootd_role, " xrootd_version ",xrootd_version) + if not 'timeout' in xrootd_role and not 'FATAL' in xrootd_role : + if not 'timeout' in xrootd_version and not 'FATAL' in xrootd_version : + updatexrdVersions ( endpoint, xrootd_role, xrootd_version ) + #if not xrootd_version in XrdVersions[endpoint]['xrootd_version'] or not xrootd_role in XrdVersions[endpoint]['xrootd_role'] : + # updatexrdVersions ( endpoint, xrootd_role, xrootd_version ) + from xrdVersions import XrdVersions + if endpoint in XrdVersions : + if 'timeout' in xrootd_role or 'FATAL' in xrootd_role : + #print ( "( endpont )",endpoint ) + xrootd_role = XrdVersions[endpoint]['Role'] + #sites[thesite]['xrootd_role'][xrd_idxes[idx]] = '['+xrootd_role+']' + #if 'timeout' in xrootd_version or 'FATAL' in xrootd_version : sites[thesite]['xrootd_version'][xrd_idxes[idx]] = '['+XrdVersions[endpoint]['Version']+']' + sites[thesite]['xrootd_role'][xrd_idxes[idx]] = xrootd_role + if 'timeout' in xrootd_version or 'FATAL' in xrootd_version : sites[thesite]['xrootd_version'][xrd_idxes[idx]] = XrdVersions[endpoint]['Version'] + + +#def updateFederation (thesite) : +# xrdInfo = sites[thesite] +# xrd_idxes = findXROOTDIdxes ( xrdInfo ) +# #print ( thesite, sites[thesite] ) +# for idx in range(len(xrd_idxes)) : +# endpoint = sites[thesite]['endpoints'][xrd_idxes[idx]] +# (err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") +# #xrdInfo['xrootd_version'][xrd_idxes[idx]] = xrootd_version +# sites[thesite]['federation'][xrd_idxes[idx]] = xrootd_version +# (err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") +# #xrdInfo['xrootd_role'][xrd_idxes[idx]] = xrootd_role +# sites[thesite]['xrootd_role'][xrd_idxes[idx]] = xrootd_role + +def updateContactInfo () : + global sites + for site in sites : + xrdInfo = sites[site] + xrd_idxes = findXROOTDIdxes ( xrdInfo ) + contact = '' + for contact in sites[site]['contact'] : + if '@' in contact : break + for idx in range(len(xrd_idxes)) : + #sites[site]['contact'][xrd_idxes[idx]] = contact + sites[site]['contact'][xrd_idxes[idx]] = contact.replace('@cern.ch','') + +def findXROOTDIdxes (thesite) : + idxes=[] + i=0 + for flavor in thesite['flavors'] : + if flavor == 'XROOTD' : idxes.append(i) + i += 1 + if len(idxes) == 0 : + i=0 + for flavor in thesite['flavors'] : + if flavor == 'WEBDAV' : idxes.append(i) + i += 1 + + return idxes +def runqueryXrdmapc () : + signal.signal(signal.SIGALRM, alarm_handler) + try: + threads = list() + for redirector in topRedirectors: + if 1 == 0 : + queryXrdmapc (redirector+':1094') + else : + t = threading.Thread(target=queryXrdmapc, args=(redirector+':1094',)) + threads.append(t) + t.start() + print ("Start joining threads ") + for xrd, thread in enumerate(threads): + #print ("Before joining thread ", site) + thread.join() + print ("All threads done ") + except Alarm: + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + #clear_lock() + #sys.exit(2) + signal.alarm(0) + +def runThreadedupdateXrdInfo () : + + signal.signal(signal.SIGALRM, alarm_handler) + try: + threads = list() + for site in sites: + if 1 == 0 : + updateXrdInfo (site) + else : + t = threading.Thread(target=updateXrdInfo, args=(site,)) + threads.append(t) + t.start() + print ("Start joining threads ") + for site, thread in enumerate(threads): + #print ("Before joining thread ", site) + thread.join() + print ("All threads done ") + except Alarm: + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + #clear_lock() + #sys.exit(2) + signal.alarm(0) + + +#with open(THEPATH+'exceptions.json') as f: exc = f.read() +#exc = json.loads(exc) + +#def exception(name): +# ret = None +# for i in exc.keys(): +# if i == name : return exc[i]['VOname'] +# return ret + + +#def siteName2CMSSiteName(dom): +# ret = None +# for cmsSite in sites.keys(): +# ret = exception(dom) +# if ret : +# ret = str(ret) +# return ret +# if sites[cmsSite]['hosts'][0].find(dom) != -1: +# return cmsSite +# return ret + +def jsonSorting(item): + if isinstance(item, dict): + return sorted((key, jsonSorting(values)) for key, values in item.items()) + if isinstance(item, list): + return sorted(jsonSorting(x) for x in item) + else: + return item + +def getAllSiteRedirectors () : + global sites + try : + print ( 'DEBUG Doing getSites' ) + sites = getSites() + + print ( 'DEBUG Doing gRedirs ') + gredirs = getGlobalRedirectors() + + print ( 'DEBUG Doing rRedirs ') + #regionalRedirectors = getRegionalRedirectorsFromFile() + regionalRedirectors = getRegionalRedirectors () + #print ( 'DEBUG regionalRedirectors ', regionalRedirectors ) + print ( 'DEBUG Doing xrdServers ') + #xrdProdServers = getXrootdServersFromFile() + xrdProdServers = getXrootdServers() + #print ( 'DEBUG xrdProdServers ', xrdProdServers ) + print ( 'DEBUG Doing xrd from trans redirector ') + #xrdTranServers = getXrootdsFromTransFile() + xrdTranServers = getTransitionalXrootds() + #print ( 'DEBUG xrdTranServers ', xrdTranServers) + print ( 'DEBUG Doing sites') + for site in sites.keys(): + #print ( 'DEBUG Doing site g ',site ) + if 'T0_CH_CERN' in site : + for sitekey in sites[site].keys() : + for siteval in gredirs[sitekey] : + sites[site][sitekey].append(siteval) + #print ( 'Doing site r ',site ) + #if 'cmsxrootd-site3.fnal.gov:1094' in sites[site]['endpoints'] : + #print ( ' before regional ', site, ' has it in ', sites[site]['endpoints'] ) + if site in regionalRedirectors : #.has_key(site): + for sitekey in sites[site].keys() : + + #print (" site RR ",site, " sitekey ",sitekey, sites[site][sitekey]) + + if 'sites' in sitekey : sites[site][sitekey].append( site ) + elif 'hosts' in sitekey : sites[site][sitekey].append( regionalRedirectors[site]['host'] ) + elif 'flavors' in sitekey : sites[site][sitekey].append( 'XROOTD' ) + elif 'endpoints' in sitekey : sites[site][sitekey].append( regionalRedirectors[site]['endpoint'] ) + elif 'name' in sitekey : sites[site][sitekey].append( 'RR_service_from_'+site ) + elif 'xrootd_version' in sitekey : + #(err_info,xrootd_version,dump_info) = xrd_info(regionalRedirectors[site]['endpoint'],"version") + sites[site][sitekey].append( '' ) + elif 'xrootd_role' in sitekey : + #(err_info,xrootd_role,dump_info) = xrd_info(regionalRedirectors[site]['endpoint'],"role") + sites[site][sitekey].append( '' ) + elif 'xrootd_servers' in sitekey : sites[site][sitekey].append( 'RR' ) + elif 'xrootd_storage' in sitekey : + if SiteStorages[ site ] : xrootd_storage = SiteStorages[ site ] + else : + xrootd_storage = getStorageFromStorageJson ( site ) + updateSiteStorage ( site , xrootd_storage ) + + sites[site][sitekey].append( xrootd_storage ) + elif 'federation' in sitekey : sites[site][sitekey].append( 'prod' ) + else : + sites[site][sitekey].append( 'RR' ) + #if 'cmsxrootd-site3.fnal.gov:1094' in sites[site]['endpoints'] : + # print ( ' after regional ', site, ' has it in ', sites[site]['endpoints'] ) + ic = 0 + for xrdServers in [ xrdProdServers, xrdTranServers ] : + #ic = ic + 1 #print ( 'Doing xrdServers ',xrdServers ) #site x ',site, ' endpoints count ', xrdServers[site]['endpoints'] ) + if site in xrdServers : #.has_key(site): + #print (' xrdServers has key ic site ',ic,site, xrdServers[site]) + for idx in range ( len ( xrdServers[site]['endpoints'] ) ) : + if xrdServers[site]['endpoints'][idx] in sites[site]['endpoints'] : + #xrdInfo = sites[site] + #if xrdServers[site]['endpoints'][idx] == 'cmsxrootd-site1.fnal.gov:1094' : print ( site, "will update sites[site]['endpoints'] for ",xrdServers[site]['endpoints'][idx], xrdInfo ) #updateFederation (site) # + #xrd_idxes = findXROOTDIdxes ( xrdInfo ) + for xrd_idx in range(len( sites[site]['endpoints'] )) : + #if xrdServers[site]['endpoints'][idx] == 'cmsxrootd-site1.fnal.gov:1094' : #updateFederation (site) # + # print ( site, "will update sites[site]['endpoints'] for ",xrdServers[site]['endpoints'][idx], ' vs ',sites[site]['endpoints'][xrd_idx]) + if xrdServers[site]['endpoints'][idx] in sites[site]['endpoints'][xrd_idx] : + #if xrdServers[site]['endpoints'][idx] == 'cmsxrootd-site1.fnal.gov:1094' : #updateFederation (site) # + # print ( site, "will update sites[site]['endpoints'] for ",xrdServers[site]['endpoints'][idx]) + #sites[site]['endpoints'][xrd_idx] = xrdServers[site]['endpoints'][idx] + #sites[site]['hosts'][xrd_idx] = xrdServers[site]['hosts'][idx] + #sites[site]['flavors'][xrd_idx] = xrdServers[site]['flavors'][idx] + sites[site]['federation'][xrd_idx] = xrdServers[site]['federation'][idx] + #print ( 'skipping site idx ',site,idx,xrdServers[site]['endpoints'][idx],sites[site]['endpoints']) + continue + #for sitekey in sites[site].keys() : + #for sitekey in sites[site].keys() : + #print ( 'Doing site x site ',site, ' n endpoints ',len ( xrdServers[site]['endpoints'] ), ' endpoint ',xrdServers[site]['endpoints'][idx]) + #for idx in range ( len ( xrdServers[site]['endpoints'] ) ) : + #if not 'sites' in sitekey : continue + #print ( 'Doing site x ',site ) + #print ( 'Doing site x site ',site, ' n endpoints ',len ( xrdServers[site]['endpoints'] ), ' idx ', idx, ' endpoint ',xrdServers[site]['endpoints'][idx]) + + sites[site]['sites'].append( site ) + sites[site]['hosts'].append( xrdServers[site]['hosts'][idx] ) + sites[site]['flavors'].append( 'XROOTD' ) + sites[site]['endpoints'].append( xrdServers[site]['endpoints'][idx] ) + sites[site]['name'].append( 'Xrootd_Servers_from_'+site ) + sites[site]['xrootd_version'].append( 'unknown' ) + sites[site]['xrootd_role'].append( 'server' ) + sites[site]['xrootd_servers'].append( xrdServers[site]['hosts'][idx] ) + xrootd_storage = 'uninitialized' + if SiteStorages[ site ] : xrootd_storage = SiteStorages[ site ] + else : + xrootd_storage = getStorageFromStorageJson ( site ) + updateSiteStorage ( site , xrootd_storage ) + + sites[site]['xrootd_storage'].append( xrootd_storage ) + sites[site]['contact'].append( site ) + sites[site]['federation'].append( xrdServers[site]['federation'][idx] ) + + #print ( 'Doing site x ',site, sites[site]) + updateContactInfo() + print ("DEBUG return sites for getAllSiteRedirectors") + return sites + except Exception as e : + err={} + err["error"] = str(e) + #print (json.dumps(err)) + print ( err ) + sys.exit(1) + +#for site in sites : +# idxes = findXROOTDIdxes (sites[site]) +# #print ( ' site ', site, ' fed ',sites [ site ]['federation'], ' endpoint ',sites [ site ]['endpoints'] ) +# for idx in idxes : +# if sites [ site ]['flavors'][idx] == 'XROOTD' : +# print ( ' site ', site, 'idx ',idx, ' fed ',sites [ site ]['federation'][idx], ' endpoint ',sites [ site ]['endpoints'][idx] ) +## if 'T2_US_Florida' in site : +## for key in sites [site] : +## print ( len (sites[site][key]), key, sites[site][key] ) + +#sys.exit(0) + +# Checks if a hostname is an alias or not +def getDNSARecords ( h ): + import dns.resolver + #import socket + if h in DNSARecords : #.has_key (h) : + if DNSARecords[h]['Alias'] == True : + return ( DNSARecords[h]['Alias'], DNSARecords[h]['A'] ) + + isAlias = False + aRecords = [] + try : + #result = dns.resolver.query(h, 'A') + result = dns.resolver.resolve(h, 'A') + for ipval in result: + ip = ipval.to_text() + (host, alias, ipl) = socket.gethostbyaddr(ip) + aRecords.append (host) + if len(aRecords) > 1 : + isAlias = True + #print ("Updating DNS Records for h = ", h) + updateDNSARecords ( h, isAlias, aRecords ) + return (isAlias, aRecords) + except Exception as e : + print ("Exception h ",h, " ",e) + return (isAlias, aRecords) + +def updateDNSARecords ( h, isAlias, aRecords ) : + #if not DNSARecords.has_key (h) : + DNSARecords[ h ] = {'Alias' : isAlias, 'A' : aRecords } + with open(THEPATH+'DNSARecords.py','w') as data: + #print ( 'DNSARecords = ' + str(DNSARecords) ) + data.write('DNSARecords = ' + str(DNSARecords)) + +def updatexrdVersions ( endpoint, role, version ) : + #if not DNSARecords.has_key (h) : + XrdVersions[ endpoint ] = {'Role' : role, 'Version' : version } + with open(THEPATH+'xrdVersions.py','w') as data: + data.write('XrdVersions = ' + str(XrdVersions)) + +class MyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, np.ndarray): + return obj.tolist() + elif isinstance(obj, bytes): + return str(obj, encoding='utf-8'); + return json.JSONEncoder.default(self, obj) + +if __name__ == "__main__": + # run xrdmapc against the global redirector and the transitional redirector + runqueryXrdmapc () + + # get all redirectors from the hardwired way or the vofeed or the xrdmapc output + sites = getAllSiteRedirectors () + + # fill xrootd version and xrootd role in a threaded way + runThreadedupdateXrdInfo () + + # + #if 1 == 0 : + for site in sites : + idxes = findXROOTDIdxes (sites[site]) + for idx in idxes : + xrdv = sites [ site ]['xrootd_version'][idx] + endpoint = sites [ site ]['endpoints'][idx] + xrdr = sites [ site ]['xrootd_role'][idx] + if 'timeout' in xrdv : + #(err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + #(err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + #sites [ site ]['xrootd_version'][idx] = xrootd_version + print ( site, endpoint,xrdv) # " version new ",xrootd_version) + if 'timeout' in xrdr : + #(err_info,xrootd_version,dump_info) = xrd_info(endpoint,"version") + #(err_info,xrootd_role,dump_info) = xrd_info(endpoint,"role") + #sites [ site ]['xrootd_role'][idx] = xrootd_role + print ( site, endpoint,xrdr) # " role new ",xrootd_role) + # + + + federations={} + for site in sites : + idxes = findXROOTDIdxes (sites[site]) + for idx in idxes : + # Update if the host is an alias and the federation is nowhere + if ( 'T0_' in site or 'T1_' in site or 'T2_' in site ) and sites [ site ]['federation'][idx] == 'nowhere' : + #print ( 'Doing getDNSRecords ', sites [ site ]['hosts'][idx] ) + (isAlias, A) = getDNSARecords ( sites [ site ]['hosts'][idx] ) + if isAlias : + aFound = True + shost = '' + for ar in A : + #print ( ' ar ',ar, ' hosts ', sites [ site ]['hosts'] ) + if not ar in sites [ site ]['hosts'] : aFound = False + shost = ar + + if len (A) > 1 : + if shost in sites [ site ]['hosts'] : + sites [ site ]['federation'][idx] = sites [ site ] ['federation'][sites [ site ]['hosts'].index ( shost )] + else : + print ( ' len(A) ', sites [site]['hosts'] ) + + #print ( ' site ', site, 'idx ',idx, ' fed ',sites [ site ]['federation'][idx], ' endpoint ',sites [ site ]['endpoints'][idx] , isAlias , aFound ) + #print ( ' site ', site, 'idx ',idx, ' fed ',sites [ site ]['federation'][idx], ' endpoint ',sites [ site ]['endpoints'][idx] ) + #output[sites[site]['federation'][idx]].append(sites[site]) + federations[sites[site]['federation'][idx]] = {} + for federation in federations : + for site in sites : + idxes = findXROOTDIdxes (sites[site]) + for idx in idxes : + if sites[site]['federation'][idx] in federation : + thesite = {} + thesite['sites'] = sites[site]['sites'][idx] + thesite['hosts'] = sites[site]['hosts'][idx] + thesite['flavors'] = sites[site]['flavors'][idx] + thesite['endpoints'] = sites[site]['endpoints'][idx] + thesite['name'] = sites[site]['name'][idx] + thesite['contact'] = sites[site]['contact'][idx] + thesite['xrootd_version'] = sites[site]['xrootd_version'][idx] + thesite['xrootd_role'] = sites[site]['xrootd_role'][idx] + thesite['xrootd_servers'] = sites[site]['xrootd_servers'][idx] + thesite['xrootd_storage'] = sites[site]['xrootd_storage'][idx] + thesite['federation'] = sites[site]['federation'][idx] + output[federation].append(thesite) + #for federation in federations : + # for thesite in output[federation] : + # print ( federation, thesite ) + with open(THEPATH+'out/federations.json', 'w') as f: + #f.write(json.dumps(output, indent = 1)) + f.write(json.dumps(output,cls=MyEncoder,indent=1)) + print ("Check ",THEPATH+'out/federations.json') +exit() diff --git a/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py b/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py new file mode 100644 index 0000000..3cb6735 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/getHostsFromAlias.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import sys +import socket +import dns.resolver + +if __name__ == "__main__": + h="xrootd-cms.infn.it" + h="cms-xrootd.gridpp.ac.uk" + if len(sys.argv) > 1 : + h=sys.argv[1] + result = dns.resolver.resolve(h, 'A') + for ipval in result: + ip = ipval.to_text() + (host, alias, ipl) = socket.gethostbyaddr(ip) + print ( host, alias, ipl ) diff --git a/AAAOps/FedProbeSendAAAMetrics/listSrv.sh b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh new file mode 100755 index 0000000..3a1f11e --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/listSrv.sh @@ -0,0 +1,9 @@ +#!/bin/bash +ipv= ; ipos=2 +[ $# -gt 0 ] && { ipv=6 ; ipos=3 ; } ; +[ -f out/xrdmapc_all_0.txt ] || exit 1 +for h in $( grep Srv out/xrdmapc_all_0.txt | awk '{print $2}' | cut -d: -f1 | grep -v \\[ | sort -u ) ; do + + echo $h $(ping${ipv} -c 1 $h | grep ^PING | cut -d\( -f${ipos} | cut -d\) -f1) + #sleep 1 +done diff --git a/AAAOps/FedProbeSendAAAMetrics/mapHostSitename.sh b/AAAOps/FedProbeSendAAAMetrics/mapHostSitename.sh new file mode 100755 index 0000000..ecdca97 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/mapHostSitename.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for h in $( grep Srv out/xrdmapc_all_0.txt | awk '{print $2}' | cut -d: -f1 | grep -v \\[ | sort -u ) ; do + python findSitename.py $h + break +done diff --git a/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh new file mode 100755 index 0000000..dbd4453 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/probe_create_send_aaa_metrics.sh @@ -0,0 +1,278 @@ +#!/bin/bash +THEPATH=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +FED_json=$THEPATH/out/federations.json +THELOG=$THEPATH/logs/probe_create_send_aaa_metrics.log +KIBANA_PAGE=https://monit-kibana.cern.ch/kibana/goto/5d1128ff8482ac3b00e4be3d5a06e954 +GRAFANA_PAGE="https://monit-grafana.cern.ch/d/5njhdTrWk/site-subscription?from=now-7d&orgId=11&to=now" + +export X509_USER_PROXY=$HOME/.globus/slsprobe.proxy +if [ -f $X509_USER_PROXY ] ; then + if [ $(voms-proxy-info -timeleft 2>/dev/null) -lt 3600 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\n$(voms-proxy-info -all | sed 's#%#%%#g')" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 1" $notifytowhom + fi +else + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $X509_USER_PROXY.\nIt does not exist" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0) proxy issue 2" $notifytowhom +fi +notifytowhom=bockjoo__AT__gmail__dot__com +export PYTHONPATH=$PYTHONPATH:$THEPATH/CMSMonitoring/src/python/ +notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') + +[ -d $THEPATH/out ] || mkdir -p $THEPATH/out + +python3 $THEPATH/create_fedmaps.py > $THEPATH/create_fedmaps.log 2>&1 +if [ $? -ne 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 $THEPATH/create_fedmaps.py\n\n$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom +fi +if [ ! -r $FED_json ]; then + echo "We have a problem creating JSON file.\n" + printf "$(/bin/hostname -s) $(basename $0) We have a problem creating JSON file.\n\nn$THEPATH/create_fedmaps.log:\n$(cat $THEPATH/create_fedmaps.log | sed 's#%#%%#g')\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + exit 1 +fi + +cat $FED_json | python -m json.tool 2>/dev/null 1>/dev/null +if [ $? -ne 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem with $FED_json.\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $FED_json + exit 1 +fi +[ -d $(dirname $THELOG) ] || mkdir -p $(dirname $THELOG) + +# Server Version List +if [ ] ; then +( +is_int() { + if [ $1 ] ; then + theval=$1 + if [ "x$theval" == "x" ] ; then + return 1 + else + if [ "x`echo $theval | egrep -v '^[0-9]+$'`" != "x" ] ; then + return 1 + else + return 0 + fi + fi + else + return 1 + fi +} + timeout=10 + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Reply-To: noreply@cern.ch" + echo "Subject: XRootD Version Role List" + echo "Content-Type: text/html" + echo "" + echo "" + echo "" + grep "\"sites\"\|\"endpoints\"\|version\|role" $FED_json | \ + while read site ; do + read endpoints + read version + read role + site=$(echo $site | cut -d\" -f4) + echo "$site" | grep -q T3 && continue + endpoints=$(echo $endpoints | cut -d\" -f4) + version=$(echo $version | cut -d\" -f4) + role=$(echo $role | cut -d\" -f4) + # port 0 is an invalid port + port=$(echo $endpoints | cut -d: -f2) ; [ $port -eq 0 ] && continue + [ "x$version" == "xtimeout" ] && version=$(echo $(perl -e "alarm $timeout ; exec @ARGV" xrdfs $endpoints query config version | grep "^v\|Alarm clock")) + [ "$version" == "Alarm clock" ] && version="$timeout(sec)TO" + [ "x$role" == "xtimeout" ] && role=$(echo $(perl -e "alarm $timeout ; exec @ARGV" xrdfs $endpoints query config role)) + [ "$role" == "Alarm clock" ] && role="$timeout(sec)TO" + bgcolor="yellow" + a=$(echo "$version" | grep v | sed "s#[a-z]##g" | sed "s#[A-Z]##g" | cut -d. -f1) + [ "x$a" == "x" ] && a=1 + [ $(is_int $a ; echo $?) -eq 0 ] || a=1 + a=$(expr 100000 \* $a) + b=$(echo "$version" | grep v | sed "s#[a-z]##g" | sed "s#[A-Z]##g" | cut -d. -f2) + [ "x$b" == "x" ] && b=1 + [ $(is_int $b ; echo $?) -eq 0 ] || b=1 + b=$(expr 1000 \* $b) + c=$(echo "$version" | grep v | sed "s#[a-z]##g" | sed "s#[A-Z]##g" | cut -d. -f3) + [ "x$c" == "x" ] && c=1 + [ $(is_int $c ; echo $?) -eq 0 ] || c=1 + c=$(expr 10 \* $c) + expr $a + $b + $c 2>/dev/null 1>/dev/null && \ + [ $(expr $a + $b + $c) -lt 504000 ] && bgcolor="red" + #[ "x$version" == "xv5.4.3" ] || [ "x$version" == "xv5.4.2" ] || bgcolor="red" + + echo "" + done + echo "
SiteEndpointVersionRole
$site $endpoints $version <"'!'"-- a=$a b=$b c=$c status=$? --> $role
" + echo "" +) | /usr/sbin/sendmail -t +fi # if [ ] ; then + +rm -f $THEPATH/fed.json +python3 $THEPATH/aaa_federation.py --amq $THEPATH/credentials.json > $THEPATH/aaa_federation.log 2>&1 +status=$? +# Check any error from xrdmapc +xrdmapc_error=$(grep \\[ $THEPATH/out/xrdmapc_prod_1.txt | grep -v "invalid addr" |sort -u) +xrdmapc_port0_error=$(grep ":0$\|:0 " $THEPATH/out/xrdmapc_prod_1.txt | awk '{if ($2 == "Man") print $3 ; else print $2}' | sed 's#:0$##g' | while read h ; do grep -v $h:0 $THEPATH/out/xrdmapc_prod_1.txt | grep -q $h $THEPATH/out/xrdmapc_prod_1.txt ; [ $? -eq 0 ] || echo ${h}:0 ; done) +if [ $status -eq 0 ] ; then + if [ $(date +%M ) -lt 15 ] ; then + : + #printf "$(/bin/hostname -s) $(basename $0) AAA metrics sent.\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -r noreply@cern.ch -s "INFO $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log + fi +else + printf "$(/bin/hostname -s) $(basename $0) There might have been an issue or more with sending AAA metrics\nSee $KIBANA_PAGE\n$GRAFANA_PAGE\nxrdmapc errors: \n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" | mail -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom -a $THEPATH/logs/probe_create_send_aaa_metrics.log +fi + +#nprod_exp=55 +nprod_exp=54 +if [ -f $THEPATH/check_subscribed_sites.sh ] ; then + subscribed_sites=$($THEPATH/check_subscribed_sites.sh) + nprod=$(printf "$subscribed_sites\n" | tail -1 | awk '{print $2}') + if [ $nprod -lt 10 ] ; then + echo Warning something is wrong + ( + echo "To: "$(echo $notifytowhom | sed "s#__AT__#@#" | sed "s#__dot__#\.#g") + echo "Reply-To: noreply@cern.ch" + echo "Subject: XRootD Version Role List" + echo "Content-Type: text/html" + echo "AAA Subscription: https://monit-grafana.cern.ch/d/5njhdTrWk/site-subscription?orgId=11
" + echo "Redirector Availability: https://monit-grafana.cern.ch/d/serviceAvailability/overview-service-availability?orgId=11&var-category=All&from=now-24h&to=now-15m%2Fm
" + echo "
"
+	 echo $THEPATH/create_fedmaps.log
+	 cat $THEPATH/create_fedmaps.log
+	 echo ; echo ; echo
+	 echo $THEPATH/fed.json
+	 cat $THEPATH/fed.json
+	 echo "
" + ) | /usr/sbin/sendmail -t + fi + printf "$subscribed_sites\n" > $THEPATH/subscribed_sites_$nprod.txt + if [ $nprod_exp -gt $nprod ] ; then + echo DEBUG nprod_exp -gt nprod : $nprod_exp -gt $nprod + thediff= + if [ -f $THEPATH/subscribed_sites_$nprod_exp.txt ] ; then + thediff=$(diff $THEPATH/subscribed_sites_$nprod.txt $THEPATH/subscribed_sites_$nprod_exp.txt | sed 's#%#%%#g' | grep T | cut -d\" -f2 | sort -u) + fi + day_of_week=$(date +%u%H%M%S) + if [ $day_of_week -gt 1014000 -a $day_of_week -lt 1002800 ] ; then + echo "" > $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt + fi + if [ ! -f $THEPATH/site_aaa_status.html ] ; then + echo "" > $THEPATH/site_aaa_status.html + echo "
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + fi + + thestring="" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="Sites with Frequent Subscription Failure" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
Site
" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html + sed -i "1s|^|\n|" $THEPATH/site_aaa_status.html + sed -i "1s/^/
Site Life Status SAM Status Down Status WkCount Expected SAM Critical
Sites with Frequent Subscription Failure
\n/" $THEPATH/site_aaa_status.html + sed -i "1s/^/\n/" $THEPATH/site_aaa_status.html + + if [ ! -f $THEPATH/site_aaa_status.txt ] ; then + printf "%20s %20s %20s %7s %7s\n" Site "Life Status" "SAM Status" WkCount Expected > $THEPATH/site_aaa_status.txt + fi + sam3result= + for thesite in $thediff ; do + $THEPATH/cms_sam3_check.sh $thesite > $THEPATH/out/cms_sam3_check.${thesite}.txt 2>&1 + status=$? + result="SAM3 OK" ; [ $status -eq 0 ] || result="SAM3 FAIL" + site_status=$(python3 -m json.tool $THEPATH/cms_sam3_check_monit_prod_cmssst_search.out | grep -A 2 $thesite | grep status | cut -d\" -f4 | head -1) + siteLifeStatus=$(export GRAFANA_VIEWER_TOKEN=$(cat $THEPATH/token.txt) ; python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $thesite) + siteDownStatus=$(export GRAFANA_VIEWER_TOKEN=$(cat $THEPATH/token.txt) ; python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $thesite "down15min") + + sam3result="$sam3result\n$thesite($site_status $result siteLifeStatus=$siteLifeStatus)\n" + expected=Yes + if [ "$result" == "SAM3 OK" ] ; then + if [ $(echo "$siteLifeStatus" | grep -q "enabled" ; echo $?) -eq 0 ] ; then + [ "x$siteDownStatus" == "xdowntime" ] || expected=No + fi + fi + sam_critical_host_tests=$(grep CRITICAL $THEPATH/out/cms_sam3_check.${thesite}.txt | awk '{print $2" "$4"
"}') + other_comments= + if [ $(grep -q "$thesite" $THEPATH/site_aaa_status.txt ; echo $?) -eq 0 ] ; then + siteline=$(grep "$thesite" $THEPATH/site_aaa_status.txt) + WkCount_previous=$(grep "$thesite" $THEPATH/site_aaa_status.txt | awk '{print $5}') + WkCount=$(expr $WkCount_previous + 1) + other_comments=$(echo $(grep "$thesite" $THEPATH/site_aaa_status.txt | grep "|" | cut -d\| -f2-)) + [ "x$other_comments" == "x" ] || other_comments=" | $other_comments" + sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.txt + printf "%20s %20s %20s %7s %7s %80s\n" "$thesite" "$siteLifeStatus" "$result" $WkCount $expected "$other_comments" >> $THEPATH/site_aaa_status.txt + sed -i "/$(echo $thesite | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + #echo DEUBG "thesite = $thesite siteLifeStatus = $siteLifeStatus result = $result WkCount = $WkCount expected = $expected sam_critical_host_test = $sam_critical_host_tests" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + else + printf "%20s %20s %20s %7s %7s\n" "$thesite" "$siteLifeStatus" "$result" 1 $expected >> $THEPATH/site_aaa_status.txt + echo "" >> $THEPATH/site_aaa_status.html + fi + done + if [ -f $THEPATH/site_aaa_status.html ] ; then + thestring="
$thesite $siteLifeStatus $result $siteDownStatus $WkCount $expected $sam_critical_host_tests $other_comments
$thesite $siteLifeStatus $result $siteDownStatus 1 $expected $sam_critical_host_tests
" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + thestring="" + sed -i "/$(echo $thestring | sed 's^/^\\\/^g')/ d" $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + echo "" >> $THEPATH/site_aaa_status.html + fi + #grep -q "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html + #if [ $? -ne 0 ] ; then + # echo DEBUG updating with "Sites with Frequent Subscription Failure" $THEPATH/site_aaa_status.html + # sed -i "1s|^| Sites with Frequent Subscription Failure \n|" $THEPATH/site_aaa_status.html + #fi + #if [ "x$thediff" == "xT2_UA_KIPT" ] ; then + #echo $sam3result | grep -q "SAM3 OK" && printf "$(/bin/hostname -s) $(basename $0) \n$(cat $THEPATH/site_aaa_status.html)\n$(cat $THEPATH/site_aaa_status.txt)\nWe have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + echo DEBUG sam3result + printf "$sam3result\n" + if [ $(echo $sam3result | grep -q "SAM3 OK" ; echo $?) -eq 0 ] ; then + ( + echo "To: $notifytowhom" + echo "Subject: $(basename $0) on $(/bin/hostname -s)" + echo "Reply-to: noreply@cern.ch" + echo "Content-Type: text/html" + cat $THEPATH/site_aaa_status.html + ) | /usr/sbin/sendmail -t + echo DEBUG sam3result OK mail sent + else + echo DEBUG sam3result not OK + fi + #else + # echo $sam3result | grep -q "SAM3 OK" && \ + # printf "$(/bin/hostname -s) $(basename $0) We have a problem with $nprod\n$sam3result\n\n$(for thesite in $thediff ; do cat $THEPATH/out/cms_sam3_check.${thesite}.txt ; done)\n" | mail -r noreply@cern.ch -s "Warn $(/bin/hostname -s) $(basename $0)" $notifytowhom + #fi + #exit 1 + else + echo INFO not "($nprod_exp -gt $nprod)" + ( + echo "To: $notifytowhom" + echo "Subject: $(basename $0) on $(/bin/hostname -s) $nprod_exp is not gt $nprod " + echo "Reply-to: noreply@cern.ch" + echo "Content-Type: text/html" + $nprod_exp is not greater than $nprod + ) | /usr/sbin/sendmail -t + fi + echo "Sites subscribed to the Production Federation: " $nprod +fi +ip_no_host=$(echo $(grep [0-9]: $THEPATH/out/xrdmapc_prod_* | awk '{print $NF}' | sort -u)) +if [ "x$ip_no_host" != "x" ] ; then + ( + echo "To: $notifytowhom" + echo "Subject: $(basename $0) on $(/bin/hostname -s) IPs without Hostnames" + echo "Reply-to: noreply@cern.ch" + echo "Content-Type: text/html" + for ip in $ip_no_host ; do echo $ip"
" ; done + ) | /usr/sbin/sendmail -t +fi +printf "$(/bin/hostname -s) $(basename $0) \nxrdmapc errors:\n$xrdmapc_error\nPort 0 Errors: ${xrdmapc_port0_error}\n" + +[ -f ${THELOG}_$(date +%Y%m%d) ] || touch ${THELOG}_$(date +%Y%m%d) +if [ -f $THELOG ] ; then + cat $THELOG >> ${THELOG}_$(date +%Y%m%d) +fi + +exit $status diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py new file mode 100644 index 0000000..50a7bac --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +import sys +import os +import requests +import json + +#def siteLifeStatus (site='T2_US_Florida', dbid=9475, gte="now-6h/h",lt="now") : +def siteLifeStatus (site='T2_US_Florida', what='sts15min', dbid=9980, gte="now-23h/h",lte="now") : + ''' + gets a site life status during last 2 hours + ''' + url = "https://monit-grafana.cern.ch/api/datasources/proxy/"+str(dbid)+"/_msearch" + index_name = "monit_prod_cms_raw_aaa-ng*" # XRD Collector Monitoring + index_name = "monit_prod_cmssst_*" # 9475 Site Status + index_name = "monit_prod_cmssst_raw_ssbmetric-*" # 9980 Site Status + payload_index_props = { + "search_type": "query_then_fetch", + "index": [ index_name ], + "ignore_unavailable":True + } + payload_query = { + "query": { + "bool": { + "must": [ + { + "match_phrase": { + "metadata.type": "ssbmetric" + } + }, + { + "match_phrase": { + "metadata.type_prefix": "raw" + } + }, + { + "match_phrase": { + "metadata.monit_hdfs_path": "sts15min" + } + } + ], + "filter": { + "range": { + "metadata.timestamp": { + "gte": "===START_TIME===", + "lt": "===END_TIME===", + "format": "epoch_second" + } + } + } + } + }, + "_source": { + "includes": [ + "metadata.timestamp", + "metadata.kafka_timestamp", + "data.name", + "data.status" + ] + }, + "size": 10000, + "sort": [ + { "metadata.timestamp": "desc" } + ] + } + payload_query["query"]["bool"]["filter"]["range"]["metadata.timestamp"]["gte"] = gte + payload_query["query"]["bool"]["filter"]["range"]["metadata.timestamp"]["lt"] = lte + # down15min or sts15min + payload_query["query"]["bool"]["must"][2]["match_phrase"]["metadata.monit_hdfs_path"] = what + #print(payload_query) + payload = json.dumps(payload_index_props) + " \n" + json.dumps(payload_query) + "\n" + headers = { + 'Content-Type': 'application/json', + 'Authorization': 'Bearer {}'.format(os.environ["GRAFANA_VIEWER_TOKEN"]) + } + result = requests.request("POST", url, headers=headers, data = payload).json() + + + #print ( 'Number of Responses ', len ( result["responses"] ) ) + response = result["responses"][0] + #print ( 'Total hits ',response['hits']['total']['value'] ) + siteStatus={} + #print ( type ( response['hits']['hits'] ) ) + for hit in response['hits']['hits'] : + #print ("Site ",hit['_source']['data']['name'], " Status ",hit['_source']['data']['status']) + siteStatus[hit['_source']['data']['name']] = hit['_source']['data']['status'] + if len(siteStatus.keys()) == 0 : + return 'UNKNOWN' + if not site in siteStatus.keys() : return 'NoKey' + return siteStatus[site] + +if __name__ == "__main__": + if len(sys.argv) > 2 : + site = sys.argv[1] + what = sys.argv[2] + status = siteLifeStatus(site=site, what=what) + print (status) + sys.exit(0) + if len(sys.argv) > 1 : + site = sys.argv[1] + status = siteLifeStatus(site=site) + print (status) + sys.exit(0) + status = siteLifeStatus() + print (status) + + diff --git a/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh new file mode 100644 index 0000000..30a12b0 --- /dev/null +++ b/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.sh @@ -0,0 +1,16 @@ +#!/bin/bash +if [ $# -lt 1 ] ; then + echo ERROR: $(basename $0) TX__YY_ZZZZ + exit 1 +fi +sites=$1 +THEPATH=/opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics +export GRAFANA_VIEWER_TOKEN=$(echo $(cat $THEPATH/token.txt)) +if [ $(echo $sites | grep -q T ; echo $?) -eq 0 ] ; then + python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $sites +else + for site in $(echo /cvmfs/cms.cern.ch/SITECONF/T[1-2]* ) ; do + s=$(echo $site | cut -d/ -f5) + echo $s LifeStatus=$(python /opt/TransferTeam/AAAOps/FedProbeSendAAAMetrics/siteLifeStatus.py $s 2>/dev/null ) + done +fi diff --git a/AAAOps/Federation/create-fedfiles.sh b/AAAOps/Federation/create-fedfiles.sh index 94158b8..a0ab436 100755 --- a/AAAOps/Federation/create-fedfiles.sh +++ b/AAAOps/Federation/create-fedfiles.sh @@ -20,12 +20,13 @@ if [ ! -r $PROD_hosts ] || [ ! -r $TRANS_hosts ]; then exit 1 fi -python /opt/TransferTeam/AAAOps/Federation/create_fedmaps_DEV.py +python /opt/TransferTeam/AAAOps/Federation/create_fedmaps.py if [ ! -r $FED_json ]; then echo "We have a problem creating JSON file.\n" exit 1 fi + cp /opt/TransferTeam/AAAOps/Federation/out/* /var/www/html/aaa-fedinfo/ exit 0; diff --git a/AAAOps/Federation/create_allow-list.sh b/AAAOps/Federation/create_allow-list.sh index 49131a3..9e0ad1b 100755 --- a/AAAOps/Federation/create_allow-list.sh +++ b/AAAOps/Federation/create_allow-list.sh @@ -4,15 +4,15 @@ BASE=/root/ FEDINFO=/opt/TransferTeam/AAAOps/Federation/ export XRD_NETWORKSTACK=IPv4 -declare -a redirectors=("xrdcmsglobal01.cern.ch:1094" "xrdcmsglobal02.cern.ch:1094" "cms-xrd-transit.cern.ch:1094") +declare -a redirectors=("cms-xrd-global01.cern.ch:1094" "cms-xrd-global02.cern.ch:1094" "cms-xrd-transit.cern.ch:1094") for j in "${redirectors[@]}";do - if [ "$j" == "xrdcmsglobal01.cern.ch:1094" ] || [ "$j" == "xrdcmsglobal02.cern.ch:1094" ]; then + if [ "$j" == "cms-xrd-global01.cern.ch:1094" ] || [ "$j" == "cms-xrd-global02.cern.ch:1094" ]; then #query european reginal redirectors xrdmapc --list all "$j" | grep -E 'xrootd.ba.infn.it|xrootd-redic.pi.infn.it|llrxrd-redir.in2p3.fr:1094' | awk '{print $3}' | cut -d ':' -f1 > $BASE/tmp_euRED_$j xrdmapc --list all "$j" | grep -E 'cmsxrootd2.fnal.gov|xrootd.unl.edu' | awk '{print $3}' | cut -d ':' -f1 > $BASE/tmp_usRED_$j for i in $(cat $BASE/tmp_euRED_$j);do - xrdmapc --list all $i:1094 > $BASE/tmp_$i + xrdmapc --list all $i:1094 > $BASE/tmp_$i cat $BASE/tmp_$i | awk '{if($2=="Man") print $3; else print $2}' | tail -n +2 >> $BASE/tmp_total_eu_$j done @@ -23,9 +23,9 @@ for j in "${redirectors[@]}";do cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u > $FEDINFO/in/prod_$j.txt - cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u >> $FEDINFO/in/prod_$j.txt + cat $BASE/tmp_total_us_$j | cut -d : -f1 | grep -v "\[" | sort -u >> $FEDINFO/in/prod_$j.txt cat $BASE/tmp_total_eu_$j | cut -d : -f1 | grep -v "\[" | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_eu_$j.allow - cat $BASE/tmp_total_us_$j | cut -d : -f1 | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_us_$j.allow + cat $BASE/tmp_total_us_$j | cut -d : -f1 | grep -v "\[" | sort -u | awk -F. '{print "*."$(NF-2)"."$(NF-1)"."$NF}' | sort -u > $FEDINFO/out/list_us_$j.allow rm hostIPv4.txt hostIPv6.txt for f in $(cat $FEDINFO/in/prod_$j.txt);do if [ "$f" != "${1#*[0-9].[0-9]}" ]; then @@ -55,17 +55,17 @@ for j in "${redirectors[@]}";do done -diff $FEDINFO/in/prod_xrdcmsglobal01.cern.ch\:1094.txt $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt +diff $FEDINFO/in/prod_cms-xrd-global01.cern.ch\:1094.txt $FEDINFO/in/prod_cms-xrd-global02.cern.ch\:1094.txt stat=$(echo $?) if [ $stat == 1 ]; then - cat $FEDINFO/in/prod_xrdcmsglobal01.cern.ch\:1094.txt $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt | sort -u > $FEDINFO/in/prod.txt - cat $FEDINFO/out/list_eu_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_eu_xrdcmsglobal02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_eu.allow - cat $FEDINFO/out/list_us_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_us_xrdcmsglobal02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_us.allow + cat $FEDINFO/in/prod_cms-xrd-global01.cern.ch\:1094.txt $FEDINFO/in/prod_cms-xrd-global02.cern.ch\:1094.txt | sort -u > $FEDINFO/in/prod.txt + cat $FEDINFO/out/list_eu_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_eu_cms-xrd-global02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_eu.allow + cat $FEDINFO/out/list_us_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_us_cms-xrd-global02.cern.ch\:1094.allow | sort -u > $FEDINFO/out/list_us.allow else - cp $FEDINFO/in/prod_xrdcmsglobal02.cern.ch\:1094.txt $FEDINFO/in/prod.txt - cp $FEDINFO/out/list_us_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_us.allow - cp $FEDINFO/out/list_eu_xrdcmsglobal01.cern.ch\:1094.allow $FEDINFO/out/list_eu.allow + cp $FEDINFO/in/prod_cms-xrd-global02.cern.ch\:1094.txt $FEDINFO/in/prod.txt + cp $FEDINFO/out/list_us_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_us.allow + cp $FEDINFO/out/list_eu_cms-xrd-global01.cern.ch\:1094.allow $FEDINFO/out/list_eu.allow fi @@ -81,12 +81,12 @@ echo "* redirect cms-xrd-transit.cern.ch+:1213" >> $FEDINFO/out/list_eu.allow #cat $FEDINFO/in/prod.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "it" || $(NF-1) == "cern" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt #Quick fix for "[" character in prod.txt -cat $FEDINFO/in/prod.txt | awk '{ if ($1 == "[") print "Unknown.Host"; else print $1;}' > $FEDINFO/in/tmp +cat $FEDINFO/in/prod.txt | awk '{ if ($1 ~ /\[+/ ) print "Unknown.Host"; else print $1;}' > $FEDINFO/in/tmp cp $FEDINFO/in/tmp $FEDINFO/in/prod.txt rm $FEDINFO/in/tmp -cat $FEDINFO/in/prod.txt | sort -u | awk -F. '{if ($NF == "uk" && $(NF-2) != "rl" || $NF == "fr" || $(NF-1) == "cern" || $(NF-1) == "fnal" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-2) == "cnaf") print $(NF-4)"."$(NF-2); else if ( $NF == "it" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-2) == "rl" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "kr") print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "be" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt +cat $FEDINFO/in/prod.txt | sort -u | awk -F. '{if ($NF == "uk" && $(NF-2) != "rl" || $NF == "fr" || $(NF-1) == "cern" || $(NF-1) == "fnal" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "it" && $(NF-2) == "cnaf") print $(NF-4)"."$(NF-2); else if ( $NF == "it" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-3) == "xrootd-vanderbilt" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $(NF-1) == "mit" ) print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "uk" && $(NF-2) == "rl" ) print $(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "kr") print $(NF-2)"."$(NF-1)"."$NF; else if ( $NF == "be" ) print $(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/prod_domain.txt cat $FEDINFO/in/trans.txt | cut -d : -f1 | sort -u | awk -F. '{if ($NF == "uk" || $NF == "fr" || $NF == "kr" || $NF == "it" || $NF == "ch" && $(NF-2) != "cnaf" ) print $(NF-2)"."$(NF-1)"."$NF; else if ($NF == "it" && $(NF-2) == "cnaf" ) print $(NF-4)"."$(NF-3)"."$(NF-2)"."$(NF-1)"."$NF; else print $(NF-1)"."$NF}' | sort -u > $FEDINFO/in/trans_domain.txt diff --git a/AAAOps/Federation/create_fedmaps_DEV.py b/AAAOps/Federation/create_fedmaps.py similarity index 95% rename from AAAOps/Federation/create_fedmaps_DEV.py rename to AAAOps/Federation/create_fedmaps.py index 798d161..dddc396 100755 --- a/AAAOps/Federation/create_fedmaps_DEV.py +++ b/AAAOps/Federation/create_fedmaps.py @@ -73,10 +73,12 @@ def siteName2CMSSiteName(dom): ret = None for cmsSite in sites.keys(): ret = exception(dom) - if ret : - ret = str(ret) - return ret - if sites[cmsSite]['hosts'][0].find(dom) != -1: return cmsSite + if ret : + ret = str(ret) + return ret + if sites[cmsSite]['hosts'][0].find(dom) != -1: + #print cmsSite + return cmsSite return ret if __name__ == "__main__": @@ -89,7 +91,6 @@ def siteName2CMSSiteName(dom): for federation in ['prod', 'trans']: for i in domains[federation]: cmsSiteName = siteName2CMSSiteName(i) - #print cmsSiteName if cmsSiteName and not cmsSiteName in output[federation]: output[federation].append(cmsSiteName) @@ -103,3 +104,4 @@ def siteName2CMSSiteName(dom): with open('/opt/TransferTeam/AAAOps/Federation/out/federations.json', 'w') as f: f.write(json.dumps(output, indent = 1)) +exit() diff --git a/AAAOps/Federation/exceptions.json b/AAAOps/Federation/exceptions.json index adfb02a..9cc0844 100644 --- a/AAAOps/Federation/exceptions.json +++ b/AAAOps/Federation/exceptions.json @@ -1,5 +1,5 @@ { - "ccxrpli001.in2p3.fr" : { + "ccxrpli003.in2p3.fr" : { "VOname" : "T1_FR_CCIN2P3" }, @@ -11,7 +11,7 @@ "VOname" : "T2_FR_IPHC" }, - "polgrfs20.in2p3.fr " : { + "polgrid4.in2p3.fr" : { "VOname" : "T2_FR_GRIF_LLR" }, @@ -19,19 +19,19 @@ "VOname" : "T2_US_Nebraska" }, - "ds-05-01.cr.cnaf.infn.it" : { + "ds-t3-01.cr.cnaf.infn.it" : { "VOname" : "T3_IT_Bologna" }, - "ds-05-02.cr.cnaf.infn.it" : { + "ds-t3-02.cr.cnaf.infn.it" : { "VOname" : "T3_IT_Bologna" }, - "ds-202-06-33.cnaf" : { + "xrootd-cms-01.cnaf" : { "VOname" : "T1_IT_CNAF" }, - "ds-202-06-37.cnaf" : { + "xrootd-cms-01.cnaf" : { "VOname" : "T1_IT_CNAF" }, @@ -43,6 +43,10 @@ "VOname" : "T2_UK_SGrid_RALPP" }, + "ic.ac.uk" : { + "VOname" : "T2_UK_London_IC" + }, + "xrootd.knu.ac.kr" : { "VOname" : "T2_KR_KNU" }, @@ -65,6 +69,43 @@ "cmsxrd.ts.infn.it" : { "VOname" : "T3_IT_Trieste" - } + }, + + "xrootd-vanderbilt.sites.opensciencegrid.org" : { + "VOname" : "T2_US_Vanderbilt" + }, + + "se9.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + + "se1.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + + "se4.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + "se8.accre.vanderbilt.edu" : { + "VOname" : "T2_US_Vanderbilt" + }, + "purdue.edu" : { + "VOname" : "T2_US_Purdue" + }, + "cscs.ch" : { + "VOname" : "T2_CH_CSCS" + }, + "csc.fi" : { + "VOname" : "T2_FI_HIP" + }, + "eoscms-qdb-250e2da131.cern.ch" : { + "VOname" : "T2_CH_CERN" + }, + "cmsxrootd-2.gridka.de" : { + "VOname" : "T1_DE_KIT" + }, + "cms-t2-se01.sdfarm.kr" : { + "VOname" : "T2_KR_KISTI" + } } diff --git a/AAAOps/RedirectorServiceAvailability/README.initial.work b/AAAOps/RedirectorServiceAvailability/README.initial.work new file mode 100644 index 0000000..11e0c70 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/README.initial.work @@ -0,0 +1,113 @@ +Failsafe Redirector Probe: Task 5 in AAA Tasks and Milestones~Ref[1]  + + +The probe is running on vocms036 as a root cronscript. + + +[1] Fernando’s Original Version of the Probe for all redirectors + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py + + +[2] Fernando’s Failsafe Redirector Probe for one redirector + +This one is implemented by Fernando with a PING check: + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON_1_TEST.py + + +[3] Fernando’s cron script + +The cron is running every 15min. The cron script, /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetric.sh, + +executes the probe for each redirector unlike the one in [1] and uploads the metric to the data source monit_idb_cmsaaa which the Grafana queries to create the Redirector Probe Dashboard~Ref[2] (Service Availability Dashboard) as follows: + +python XRDFED-kibana-probe_JSON_1.py + +python3 send_metrics.py > $logs/XRDFED_send.log 2>&1 + +The first line probes one redirector and creates one json file + +The second line sends the json file to the MONIT data source monit_idb_cmsaaa. + +These two lines are repeated for all redirectors (14 of them as of Aug. 5, 2021). + + +[4] Failsafe Redirector Probe + +The probe in [1] was taken and the ping check in [2] is added: + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON_General.py + + +For the redirectors that fail a ping command, the probe produces an output similar to this json: + +{"status": "unavailable", "comment": "Unpingable host", "service": "XRDFED_CMS-GLOBAL01-NEW", "host": "cmxrd-global01.cern.ch", "version": "unavailable", "xrdcp_above_time": 0, "xrdcp_below_time": 0, "Comment": "N/A"} + +The json is sent as the probed metrics to the MONIT data source. + +For the redirectors that can be ping-able, the probe checks the needed metrics to send to the MONIT data source. + + +In the end, the python probe produces a file that contains 14 lines of metrics in the json format. + +The execution time of the probe is typically around 2min. + + +[5] The metric uploader + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/send_metrics.py sends the output produced by the python probe. The send_metrics.py python script is the same and used without any change. + + +[6] Cron Script + +The cron script is renamed to: + +/opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetricGeneral.sh + +The script basically contains these two lines of commands: + +python /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 + +python3 /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/send_metrics.py > $logs/XRDFED_send.log 2>&1 + + +[7] Puppetized Cron + +The cron can be puppetized by adding a cron entry that contains various crontab parameters for the necessary cron script,  /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetricGeneral.sh + +, execution using https://gitlab.cern.ch/ai/it-puppet-hostgroup-vocms/-/blob/qa/code/manifests/xrootd/monitor.pp  + +After the puppet agent is run, an entry in the crontab as follows: + +\# Puppet Name: AAA Service Availability Grafana Dashboard metric upload + +*/15 * * * * /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/uploadmetricGeneral.sh > /opt/TransferTeam.Jhonatan/AAAOps/XfedKibana_JSON/logs/uploadmetricGeneral.log 2>&1 + +will appear. + +[8] Gitlab +https://gitlab.cern.ch/bockjoo/aaa-monitoring/-/tree/master/RedirectorServiceAvailability + + +[9] Questions + +Do we need to send an alert email for extendedly unavailable redirectors? + +How to define ‘extendedly unavailable redirectors’? + +Is the Dashboard good enough for the service availability? + + +[10] Summary + +The work started by Fernando is completed by merging the ping check capability and consolidating the probe python code in a single file. The Grafana shows the Dashboard correctly. + + + +References +[0] ... +[1] ... +[2] https://monit-grafana.cern.ch/d/serviceAvailability/overview-service-availability?orgId=11&var-category=All + +[3] https://gitlab.cern.ch/ai/it-puppet-hostgroup-vocms/-/blob/qa/code/manifests/xrootd/monitor.pp diff --git a/AAAOps/RedirectorServiceAvailability/README.md b/AAAOps/RedirectorServiceAvailability/README.md new file mode 100644 index 0000000..93a4795 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/README.md @@ -0,0 +1,19 @@ +# Service Availabilty Probe and Uploader for Xrootd Redirectors + +Introduction +--- + +- This directory contains the xrootd redirector probe and the metric uploader. +- Currently the probe and the uploader run on one of the AAA vocms machines +- Details can be viewed from https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsCentralServicesXrootd and https://twiki.cern.ch/twiki/bin/view/CMSPublic/CompOpsAAAOperationsGuide#Monitoring + +Deploying the probe, the uploader, and the cron +--- +In the VM machine, as the root user, the github repository can be checked out under /opt/TransferTeam. +The crontab is managed by the puppet: https://gitlab.cern.ch/ai/it-puppet-hostgroup-vocms/-/blob/qa/code/manifests/xrootd/monitor.pp + +Operations +--- + +The probe and the uploader are executed every 15 minute by cron. +The uploaded metric is visualized in the Grafana dashboard: https://monit-grafana.cern.ch/goto/Q2CH71M7z diff --git a/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py new file mode 100644 index 0000000..39bb6e8 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/XRDFED-kibana-probe_JSON_General.py @@ -0,0 +1,420 @@ +#!/usr/bin/python3 +# functional probe and SLS extractor for the "federation" xroot services +# highlights: +# - stateless (i.e. run from cron whenever needed) +# - will try to prevent parallel runs via lockfile +# - multithreaded, one thread per service to be tested +# - overall runtime cap at 10min +# - could extract some statistics from xroot directly, but these are ever-increasing counters +# Problems: +# - need to update the code whenever a service is addded/deleted/changed +# - uses "random" files on various Xroot services all over the world, these are (for now) the same as used by the experiments but these might change.. +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +#import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import json +import shutil +import multiprocessing +html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here +#Bockjoo Uncomment and comment a line below +LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' +if 'ufhpc' in socket.gethostname() : + LOCKFILE='/opt/TransferTeam/AAAOps/RedirectorServiceAvailability/var/lock/subsys/xrdfed-kibana-probe-general' +# Bockjoo Uncomment and comment a line below +#probes_json='KIBANA_PROBES.json' +probes_json='KIBANA_PROBES_GENERAL.json' +timeout_sec = 10 * 60 +class Alarm(Exception): + pass +def alarm_handler(signum, frame): + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + clear_lock() + sys.exit(2) + raise Alarm +def clear_lock(): + try: + os.unlink(LOCKFILE) + except Exception as e: + print ("could not remove lockfile:"+str(e)) +def env_setup(): + #os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' + #os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' + os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + #os.environ['X509_USER_PROXY']='/root/.globus/.proxy' + if 'ufhpc' in socket.gethostname() : os.environ['X509_USER_PROXY']=os.environ['HOME']+'/.cmsuser.proxy' + os.environ['KRB5CCNAME']='FILE:/dev/null' + os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" +def get_proxy(): + dev_null = open('/dev/null', 'rw') + (proxyfd,proxy)=tempfile.mkstemp(prefix='x509_xrdfed_',suffix='.pem') + os.close(proxyfd) + os.environ['X509_USER_PROXY']=proxy + ret = subprocess.call(['grid-proxy-init','-pwstdin'],stdin=dev_null,) + if ret > 0: + raise Exception("Cannot get X509 proxy") + dev_null.close() +def cleanup_proxy(): + try: + os.unlink(os.environ['X509_USER_PROXY']) + except Exception as e: + print ("could not remove proxy file:"+str(e)) +def try_lock(): + ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) + if ret > 0: + print ("could not create lockfile") + return False + return True + +def check_ping(hostname): + with open(os.devnull, 'w') as DEVNULL: + try: + subprocess.check_call( + ['ping', '-c', '3', hostname], + stdout=DEVNULL, # suppress output + stderr=DEVNULL + ) + is_up = True + except subprocess.CalledProcessError: + is_up = False + return is_up + +def prepare_dictionary(servicename,redirector): + is_up = check_ping(redirector[:redirector.find(':')]) + dic={'service':servicename, 'host': redirector[:redirector.find(':')]} + print(servicename," redirector= ",redirector," is up ",is_up) + if is_up == False: + dic['version'] = 'unavailable' + dic['status'] = 'unavailable' + #dic['role'] = 'unavailable' + dic['comment'] = "Unpingable host" + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + else: + # OLD (errtext,version,out) = xrd_info(redirector) + (errtext,version,out) = xrd_info(redirector,'version') # NEW + if(errtext): + dic['version'] = 'unavailable' + dic['status'] = 'unavailable' + errtext = errtext.replace("'", "") + errtext = errtext.replace('"', '') + dic['comment'] = "Error getting info from redirector: "+errtext + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + else: + dic['version'] = version + #(errtext,role,out) = xrd_info(redirector,'role') + #if(errtext): + # dic['role'] = 'unavailable' + # dic['status'] = 'unavailable' + # errtext = errtext.replace("'", "") + # errtext = errtext.replace('"', '') + # dic['comment'] = "Error getting info from redirector: "+errtext + # dic["xrdcp_below_time"] = 0 + # dic["xrdcp_above_time"] = 0 + #else: + # dic['role'] = role + + return dic +def xrdmapc_test(redirector): + (errtext,out,err,elapsed) = run_xrd_commands("xrdmapc", ["--list","all", redirector], 180) + #print ("DEBUG xrd_test ",redirector," errtext ",errtext, " err ", err ) + return (errtext,err,elapsed) +def xrdcp_test(redirector,file): + (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", + ["-d","1", + "-f", + "-DIReadCacheSize","0", + "-DIRedirCntTimeout","180", + "root://"+redirector+'/'+file, + '/dev/null'],180) + #print ("DEBUG xrd_test ",redirector," errtext ",errtext, " err ", err ) + return (errtext,err,elapsed) +def xrd_info(redirector,what): + theargs = [redirector, "query", "config", what] + if 'version' in what or 'role' in what : command = 'xrdfs' + else : + command = 'xrdmapc' + theargs = [ '--list', 'all', redirector ] + config_out = "unknown" + count = 0 + count_limit = 3 + timelimit = 2 + if 'xrdmapc' in command : + timelimit = 90 + count_limit = 1 + while (count < count_limit): + (errtext,out,err,elapsed) = run_xrd_commands(command, theargs, timelimit) #"xrdfs", + #[redirector, + # "query","config", # 1:kXR_QStats + # what]) # a_ll stats + + if 'xrdmapc' in command : + if out : + errtext = err + config_out = out + break + if err: + #errtext = '' + #config_out = 'error' + config_out = err.replace(b'\n',b'').decode() + else: + #config_out=out + #if 'xrdmapc' in command : + # break + #else : + config_out = out.replace(b'\n',b'').decode() + # # to check if count matters config_out=config_out+'+'+str(count) + if out.replace(b'\n',b'') : break + time.sleep(1) + count += 1 + #print ( 'DEBUG count = ',count,' config_out ',config_out, ' redirector ',redirector) + if not 'xrdmapc' in command : + if not config_out : config_out = 'timeout' + if 'Auth failed' in config_out : config_out = 'Auth failed' + return (errtext,config_out,out) + +def run_xrd_commands(cmd,args,timelimit): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + if 'xrdcp' in cmd : + xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + "-DIConnectTimeout","30", + "-DITransactionTimeout","60", + "-DIRequestTimeout","60" ] + args + else : + xrd_args = [ 'perl','-e',"alarm "+str(timelimit)+" ; exec @ARGV", cmd ] + args + + + #if 'xrdmapc' in cmd : + # xrd_args = [ cmd, ] + args + try: + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + ret = proc.returncode + elapsed = (time.time() - start) + #print ( ' out ',out ) + #if 'xrdmapc' in cmd : return ('',out,err,elapsed) + err_redir_index = err.rfind(b'Received redirection to') + err_index3010 = err.rfind(b'(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind(b'(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + #print ("errtxt will be empty because ",err) + errtxt = '' + else: + #print ("errtxt will not be empty because ret ",ret," err ",err) + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind(b'Last server error') + if err_index >= 0: + err_end_index=err.find(b"\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + #print ("errtxt will not be empty because ret ",ret," err_index ", err_index, " err ",err) + except Exception as e : + errtext = errtxt + "Exception: "+str(e) + out = 'Try did not work :O' + print(out) + dev_null.close() + + return (errtxt,out,err,elapsed) + +def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): + servicename=servicename.upper() + notes_text = "Redirector: "+redirector + availability = 'Available' + availinfo = '' + c = '' + if 'status' in dicci and dicci['status'] == 'unavailable': + availability = dicci['status'] + #pass + elif file_below == None and file_above == None: + availability = 'Unavialable' + c = 'Non-existing File Above and File Below.' + dicci['xrdcp_below_time'] = 0 + dicci['xrdcp_below_time'] = 0 + else: + if (file_below): + notes_text = notes_text + "File below: " + file_below + (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) + #print ("DEBUG FILE_BELOW Test redirector ",redirector," err_below ",err_below) + if err_below: + availability = 'Degraded' + #availinfo=availinfo+" Error below redirector "+err_below + dump_sane = re.sub(b'---*',b'__',dump_below) + c = c+"Error for file BELOW: "+err_below+". Dumpsane: "+str(dump_sane.decode())+ '.' + dicci['xrdcp_below_time'] = 0 + else: + #availinfo=availinfo+" File below: OK " + dicci['xrdcp_below_time'] = elapsed_below + else: + c = "Error for file BELOW: Non-existing File Below. " + dicci['xrdcp_below_time'] = 0 + if(file_above): + notes_text = notes_text + "File elsewhere: " + file_above + (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) + #print ("DEBUG FILE_ABOVE Test redirector ",redirector," err_above ",err_above) + if err_above : + availability = 'Degraded' + #availinfo=availinfo+" Error above redirector "+err_above + dump_sane = re.sub(b'---*',b'__',dump_above) + c = c+"Error for file ABOVE: "+err_above+". Dumpsane: "+str(dump_sane.decode())+'.' + dicci['xrdcp_above_time'] = 0 + else: + #availinfo = availinfo+" File above: OK " + dicci['xrdcp_above_time'] = elapsed_above + else: + c = c + "Error for file ABOVE: Non-existing File Above." + dicci['xrdcp_above_time'] = 0 + #availinfo = availinfo + " " + notes_text + dicci['status']= str(availability) + #print ("DEBUG redirector ",redirector, " dicci status ",dicci['status'], dicci['role'],dicci['version']) + print ("DEBUG redirector ",redirector, " dicci status ",dicci['status'], dicci['version']) + #01OCT2021 if c == '': + #01OCT2021 c = 'N/A' + if c == '': + c = 'No Comments' + c = c.replace("\n", "") + c = c.replace("\r", "") + dicci ['Comment'] = c + + print ("DEBUG rdir ", redirector," dicci['Comment'] ",dicci['Comment']) + + # xrdmapc test 08SEP2022 + (err_xrdmapc,dump_xrdmapc,elapsed_xrdmapc) = xrdmapc_test(redirector) + print ("DEBUG rdir ", redirector, " xrdmapc err ",err_xrdmapc, " dump ",dump_xrdmapc, " elapsed ",elapsed_xrdmapc) + + with open(html_dir + probes_json, 'a') as f: + json.dump(dicci, f) + f.write('\n') + +def main(): + debug = 0 + if not 'ufhpc' in socket.gethostname(): + atexit.register(clear_lock) + if len(sys.argv) > 1: + if sys.argv[1] == '-d': + debug=1 + if not 'ufhpc' in socket.gethostname(): + if not try_lock(): + sys.exit(1) + if not os.path.exists(html_dir): + os.makedirs(html_dir) + env_setup() + timeout_sec = 10 * 60 # limit overall runtime to 10min + signal.signal(signal.SIGALRM, alarm_handler) + ATLASLINK="%BR%Monitoring:%BR%\n http://atl-prod07.slac.stanford.edu:8080/display?page=xrd_report/aggregated/total_xrootd_lgn %BR%\n http://dashb-atlas-xrootd-transfers.cern.ch/ui %BR%\nhttp://dashb-atlas-ssb.cern.ch/dashboard/request.py/siteview#currentView=FAX+redirectors&highlight=false %BR%\n" + CMSLINK="%BR%Monitoring:%BR%\n http://xrootd.t2.ucsd.edu/dashboard/ %BR%\n http://dashb-cms-xrootd-transfers.cern.ch/ui %BR%\n" + FILEABOVE="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + services = { + "XRDFED_CMS-GLOBAL01":{'redirector':'cms-xrd-global01.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL02":{'redirector':'cms-xrd-global02.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-UNL":{'redirector':'xrootd.unl.edu:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + #"XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', + # 'file_below': FILEBELOW, + # 'file_above': FILEABOVE, + # 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-PISA":{'redirector':'xrootd-redic.pi.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL":{'redirector':'cms-xrd-global.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US":{'redirector':'cmsxrootd.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU":{'redirector':'xrootd-cms.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-IPv6":{ 'redirector':'xrootd-cms-redir-int.cr.cnaf.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT":{'redirector':'cms-xrd-transit.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT01":{'redirector':'vocms031.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT02":{'redirector':'vocms032.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK} + } + signal.alarm(timeout_sec) + try: + diccionaries = [] + for xrd in services: + services[xrd].update(servicename=xrd) + servicename = xrd + dicci = prepare_dictionary(servicename, services[xrd]['redirector']) + diccionaries.append(dicci) + for dicci in diccionaries: + service = dicci['service'] + argus = services[service] + argus['dicci'] = dicci + if debug: + test_redirector(** argus) + else: + t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" + t.start() + #t.join() + #os.system('source ~/single_quotes.sh') + #print(diccionaries) + except Alarm: + print ("ERROR: caught overall timeout after "+str(timeout_sec)+"s\n") + clear_lock() + sys.exit(2) + print ("Check ",html_dir+probes_json) + signal.alarm(0) +if __name__ == '__main__': + for file in os.listdir(html_dir): + if file == probes_json: + os.remove(html_dir+probes_json) + break + #p = multiprocessing.Process(target=main) + #p.start() + #p.join(180) + + #if p.is_alive(): + # print"killing process" + # os._exit(1) + #p.terminate() + #p.join() + main() diff --git a/AAAOps/RedirectorServiceAvailability/logs/XRDFED_send.log b/AAAOps/RedirectorServiceAvailability/logs/XRDFED_send.log new file mode 100644 index 0000000..db6168f --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/logs/XRDFED_send.log @@ -0,0 +1,2 @@ +Opening and sending documents +Done diff --git a/AAAOps/RedirectorServiceAvailability/send_metrics.py b/AAAOps/RedirectorServiceAvailability/send_metrics.py new file mode 100644 index 0000000..10e3260 --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/send_metrics.py @@ -0,0 +1,83 @@ +#python3 + +# For a file where each line is a json valid document +# send those documents to metrics. +# Original script for AAAops + +import sys +import json +import requests +import traceback +import time +from datetime import datetime + +html_dir = '/var/www/html/aaa-probe/' +probes_json='KIBANA_PROBES_GENERAL.json' + +def update_dic_metrics(dic,metric_name,timestamp): + dic.update({'producer':'cmsaaa'}) + dic.update({'type':metric_name}) + dic.update({'timestamp':timestamp}) + + if dic.get('status')=='Available' and dic.get('version')!='': + dic.update({'codeStatus':1})#Available + + if dic.get('status')=='Unavailable' or dic.get('status')=='Degraded': + if dic.get('version')=='': + dic.update({'codeStatus':0})#Unavailable + else: + dic.update({'codeStatus':2})#Degraded + dic.update({'catCode':''}) + if dic.get('host')=='cms-xrd-global.cern.ch' or dic.get('host')=='cmsxrootd.fnal.gov' or dic.get('host')=='cms-xrd-transit.cern.ch': + dic.update({'catCode':0}) + + if dic.get('host')=='cms-xrd-global01.cern.ch' or dic.get('host')=='cms-xrd-global02.cern.ch': + dic.update({'catCode':1}) + + if dic.get('host')=='cmsxrootd2.fnal.gov' or dic.get('host')=='xrootd.unl.edu': + dic.update({'catCode':2}) + + #if dic.get('host')!='cms-xrd-global.cern.ch' or dic.get('host')!='cms-xrd-transit.cern.ch' or dic.get('host')!='cms-xrd-global01.cern.ch' or dic.get('host')!='cms-xrd-global02.cern.ch' or dic.get('host')!='cmsxrootd2.fnal.gov' or dic.get('host')!='xrootd.unl.edu': + #dic.update({'catCode':3}) + + if dic.get('host')=='vocms031.cern.ch' or dic.get('host')=='vocms032.cern.ch': + dic.update({'catCode':4}) + + if dic.get('catCode')=='': + dic.update({'catCode':3}) + + dic.update({'idb_fields':['codeStatus','status','Comment','version','service','host','xrdcp_below_time','xrdcp_above_time']}) + dic.update({'idb_tags':['status','host','codeStatus','catCode']}) + + +def send(document): + return requests.post('http://monit-metrics:10012/', data=json.dumps(document), headers={ "Content-Type": "application/json; charset=UTF-8"}) + +def send_and_check(document, should_fail=False): + response = send(document) + assert( (response.status_code in [200]) != should_fail), 'With document: {0}. Status code: {1}. Message: {2}'.format(document, response.status_code, response.text) + +def openjson_send(path,filename,metric_name,timestamp): + with open(path+filename) as jsonfile: + for line in jsonfile: + dic_line=(json.loads(line)) + update_dic_metrics(dic_line,metric_name,timestamp) + send_and_check(dic_line) + #print(dic_line) + +def main(): + metric_name="infrastructure" + now=datetime.now() + timestamp=int(datetime.timestamp(now))*1000 + + try: + print("Opening and sending documents") + openjson_send(html_dir,probes_json,metric_name,timestamp) + print("Done") + + except Exception: + print("Exception in checker procedure:") + traceback.print_exc() + sys.exit(2) +if __name__=="__main__": + sys.exit(main()) diff --git a/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh new file mode 100755 index 0000000..5a1603b --- /dev/null +++ b/AAAOps/RedirectorServiceAvailability/uploadmetricGeneral.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +#notifytowhom=cms-comp-ops-transfer-team@cernNOSPAMPLEASE.ch,bockjoo@gmailNOSPAMPLEASE.com +notifytowhom=bockjoo__AT__gmail__dot__com +notifytowhom=$(echo $notifytowhom | sed 's#__AT__#@#' | sed 's#__dot__#\.#') +GRAFANA_PAGE="https://monit-grafana.cern.ch/d/serviceAvailability/overview-service-availability?orgId=11&var-category=All&from=now-24h&to=now-15m%2Fm" + +DATE=$(date) +echo $DATE +echo "ExecutingScript" + +cd /opt/TransferTeam/AAAOps/RedirectorServiceAvailability +[ -d logs ] || mkdir -p logs +logs=logs +thelog=$(pwd)/logs/uploadmetricGeneral.log + +# General script +date +echo INFO sending the previous /var/www/html/aaa-probe/KIBANA_PROBES_GENERAL.json so that we can have a regular 15th minute entry for Grafana +python3 send_metrics.py > $logs/XRDFED_send.log 2>&1 +echo INFO content of logs/XRDFED_send.log +cat $logs/XRDFED_send.log +date + +echo INFO executing XRDFED-kibana-probe_JSON_General.py +python3 XRDFED-kibana-probe_JSON_General.py > $logs/XRDFED_probe_json.log 2>&1 +status=$? +if [ $status -ne 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have a problem in running python3 XRDFED-kibana-probe_JSON_General.py\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom +fi + +if [ ] ; then +rdirs=$(grep redirector $logs/XRDFED_probe_json.log | awk '{print $3}') +rdirs_degraded= +for r in $rdirs ; do + grep redirector $logs/XRDFED_probe_json.log | grep "$r" | grep -q Available # Degraded # Available + if [ $? -ne 0 ] ; then + #grep redirector $logs/XRDFED_probe_json.log | grep "$r" | grep -q Degraded # Available # Degraded + #if [ $? -eq 0 ] ; then + rdir_degraded=$(grep redirector $logs/XRDFED_probe_json.log | grep "$r" | awk '{print $(NF-1)"+"$NF}') + rdirs_degraded="$rdirs_degraded ${r}+$rdir_degraded" + #else + #3fi + fi +done +if [ "x$rdirs_degraded" != "x" ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have one or more degraded redirectors\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdcp -d 1 -f -DIReadCacheSize 0 -DIRedirCntTimeout 180 -DIConnectTimeout 30 -DITransactionTimeout 60 -DIRequestTimeout 60 root://rdir//store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root /dev/null\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom +fi +fi + +# check the xrdmapc Operation expired error +for r in $rdirs ; do + grep xrdmapc $logs/XRDFED_probe_json.log | grep "$r" | grep -q "\[ERROR\] Operation expired" + if [ $? -eq 0 ] ; then + printf "$(/bin/hostname -s) $(basename $0) We have the Operation expired with $r\n\n$logs/XRDFED_probe_json.log:\n$(cat $logs/XRDFED_probe_json.log | sed 's#%#%%#g')\nTry this:\nperl -e \"alarm 180 ; exec @ARGV\" xrdmapc --list all $r\n" | mail -r noreply@cern.ch -s "ERROR $(/bin/hostname -s) $(basename $0)" $notifytowhom + fi +done +echo INFO $logs/XRDFED_probe_json.log +cat $logs/XRDFED_probe_json.log +date +echo INFO running send_metrics.py +python3 send_metrics.py > $logs/XRDFED_send.log 2>&1 +echo INFO content of logs/XRDFED_send.log +cat $logs/XRDFED_send.log +date +echo INFO Done + +if [ -f $thelog ] ; then + a=1 + [ $status -eq 0 ] && { grep -q -i "caught overall timeout" $logs/XRDFED_probe_json.log ; [ $(expr $a + $?) -eq $a ] && status=1 ; } ; + if [ $status -ne 0 ] ; then + printf "$(/bin/hostname) $(basename $0)\n$(date)\n$(ls -al $thelog )\n$(cat $thelog)\n" | mail -r noreply@cern.ch -s "$(/bin/hostname) $(basename $thelog)" $(echo $notifytowhom | sed 's#NOSPAMPLEASE##g') + fi +fi + diff --git a/AAAOps/StorageBackend/checkVersion.py b/AAAOps/StorageBackend/checkVersion.py index 87f3c43..bb65ccc 100644 --- a/AAAOps/StorageBackend/checkVersion.py +++ b/AAAOps/StorageBackend/checkVersion.py @@ -4,7 +4,7 @@ with open("cms_site_names.txt", 'r') as f: - for site in f: - runCommand = "./storage_backend.sh "+site - p1 = Popen(runCommand, shell=True) - p1.wait() + for site in f: + runCommand = "./storage_backend.sh "+site + p1 = Popen(runCommand, shell=True) + p1.wait() diff --git a/AAAOps/XfedKibana/XRDFED-kibana-probe.py b/AAAOps/XfedKibana/XRDFED-kibana-probe.py index baa2ebe..e924123 100755 --- a/AAAOps/XfedKibana/XRDFED-kibana-probe.py +++ b/AAAOps/XfedKibana/XRDFED-kibana-probe.py @@ -151,6 +151,8 @@ def run_xrd_commands(cmd,args): "-DIConnectTimeout","30", "-DITransactionTimeout","60", "-DIRequestTimeout","60" ] + args + if not out: + out = "1" try: start = time.time() proc = subprocess.Popen(xrd_args, @@ -320,16 +322,26 @@ def main(): FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" services = { - "XRDFED_CMS-GLOBAL01":{'redirector':'xrdcmsglobal01.cern.ch:1094', + "XRDFED_CMS-GLOBAL01-NEW":{'redirector':'cms-xrd-global01.cern.ch:1094', 'file_below': FILEABOVE, 'file_above': FILEBELOW, 'extra_notes':CMSLINK}, - "XRDFED_CMS-GLOBAL02":{'redirector':'xrdcmsglobal02.cern.ch:1094', + "XRDFED_CMS-GLOBAL02-NEW":{'redirector':'cms-xrd-global02.cern.ch:1094', 'file_below': FILEABOVE, 'file_above': FILEBELOW, 'extra_notes':CMSLINK}, +# "XRDFED_CMS-GLOBAL01":{'redirector':'xrdcmsglobal01.cern.ch:1094', +# 'file_below': FILEABOVE, +# 'file_above': FILEBELOW, +# 'extra_notes':CMSLINK}, + +# "XRDFED_CMS-GLOBAL02":{'redirector':'xrdcmsglobal02.cern.ch:1094', +# 'file_below': FILEABOVE, +# 'file_above': FILEBELOW, +# 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', 'file_below': FILEABOVE, 'file_above': FILEBELOW, diff --git a/AAAOps/XfedKibana/kibana_cron.sh b/AAAOps/XfedKibana/kibana_cron.sh index d11bdef..22e2627 100755 --- a/AAAOps/XfedKibana/kibana_cron.sh +++ b/AAAOps/XfedKibana/kibana_cron.sh @@ -6,6 +6,9 @@ /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-US.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-EU.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log +/usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL01-NEW.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log +/usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL02-NEW.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log + /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL01.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-GLOBAL02.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log /usr/bin/curl -i -F file=@/var/www/html/aaa-probe/XRDFED_CMS-EU-LLR.xml xsls.cern.ch 2>&1 | /bin/awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; }' >> /var/log/XRDFED-kibana-xml-push.log diff --git a/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py new file mode 100644 index 0000000..2324222 --- /dev/null +++ b/AAAOps/XfedKibana_JSON/XRDFED-kibana-probe_JSON.py @@ -0,0 +1,310 @@ +#!/usr/bin/python +# functional probe and SLS extractor for the "federation" xroot services +# highlights: +# - stateless (i.e. run from cron whenever needed) +# - will try to prevent parallel runs via lockfile +# - multithreaded, one thread per service to be tested +# - overall runtime cap at 10min +# - could extract some statistics from xroot directly, but these are ever-increasing counters +# Problems: +# - need to update the code whenever a service is addded/deleted/changed +# - uses "random" files on various Xroot services all over the world, these are (for now) the same as used by the experiments but these might change.. +import xml.dom.minidom +import subprocess +import os +import sys +import signal +import re +import time +import Lemon.XMLAPI +import socket +import atexit +import threading +import tempfile +import json +import shutil +html_dir = '/var/www/html/aaa-probe/' # will create per-service json files here +LOCKFILE='/var/lock/subsys/xrdfed-kibana-probe' +class Alarm(Exception): + pass +def alarm_handler(signum, frame): + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + raise Alarm +def clear_lock(): + try: + os.unlink(LOCKFILE) + except Exception,e: + print "could not remove lockfile:"+str(e) +def env_setup(): + os.environ['X509_USER_CERT']='/root/.globus/slsprobe-cert.pem' + os.environ['X509_USER_KEY']='/root/.globus/slsprobe-key.pem' + os.environ['X509_USER_PROXY']='/root/.globus/slsprobe.proxy' + os.environ['KRB5CCNAME']='FILE:/dev/null' + os.environ['PATH']=os.environ['PATH']+":/opt/globus/bin/" +def get_proxy(): + dev_null = open('/dev/null', 'rw') + (proxyfd,proxy)=tempfile.mkstemp(prefix='x509_xrdfed_',suffix='.pem') + os.close(proxyfd) + os.environ['X509_USER_PROXY']=proxy + ret = subprocess.call(['grid-proxy-init','-pwstdin'],stdin=dev_null,) + if ret > 0: + raise Exception("Cannot get X509 proxy") + dev_null.close() +def cleanup_proxy(): + try: + os.unlink(os.environ['X509_USER_PROXY']) + except Exception,e: + print "could not remove proxy file:"+str(e) +def try_lock(): + ret = subprocess.call(['lockfile','-5','-r2',LOCKFILE]) + if ret > 0: + print "could not create lockfile" + return False + return True +def prepare_dictionary(servicename,redirector): + (errtext,version,out) = xrd_info(redirector) + dic={'service':servicename, 'host': redirector[:redirector.find(':')]} + if(errtext): + dic['version'] = 'unavailable' + dic['status'] = 'unavailable' + errtext = errtext.replace("'", "") + errtext = errtext.replace('"', '') + dic['comment'] = "Error getting info from redirector: "+errtext + dic["xrdcp_below_time"] = 0 + dic["xrdcp_above_time"] = 0 + else: + dic['version'] = version + return dic +def xrdcp_test(redirector,file): + (errtext,out,err,elapsed) = run_xrd_commands("xrdcp", + ["-d","1", + "-f", + "-DIReadCacheSize","0", + "-DIRedirCntTimeout","180", + "root://"+redirector+'/'+file, + '/dev/null']) + return (errtext,err,elapsed) +def xrd_info(redirector): + version = "(unknown)" + (errtext,out,err,elapsed) = run_xrd_commands("xrdfs", + [redirector, + "query","config", # 1:kXR_QStats + "version"]) # a_ll stats + if not out: + errtext = '' + os.system("xrdfs "+ redirector+" query config version > /root/aux.txt") + os.system("head -n 1 /root/aux.txt > /root/aux2.txt") + f = open('/root/aux2.txt', 'r') + version = f.read() + if not version: + version = "(unknown)" + else: + version = version[:-1] + else: + if not errtext: + try: + dom = xml.dom.minidom.parseString(out) + root_node = dom.documentElement + if root_node.tagName == 'statistics': + v_attr = root_node.getAttributeNode('ver') + version = v_attr.nodeValue + except Exception,e: + errtext = "ERROR: cannot parse answer:"+str(e) + return (errtext,version,out) +def run_xrd_commands(cmd,args): + dev_null = open('/dev/null', 'r') + errtxt = '' + elapsed = -1.0 + xrd_args = [ 'perl','-e',"alarm 180; exec @ARGV", cmd, # one-line wrapper that *actually* kills the command + "-DIConnectTimeout","30", + "-DITransactionTimeout","60", + "-DIRequestTimeout","60" ] + args + err = '' + out = '' + try: + ran_try = True + start = time.time() + proc = subprocess.Popen(xrd_args, + stdin=dev_null, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (out, err) = proc.communicate() + ret = proc.returncode + elapsed = (time.time() - start) + err_redir_index = err.rfind('Received redirection to') + err_index3010 = err.rfind('(error code: 3010') # (permission denied) may be sort-of-OK - we are talking to final storage already - UK + err_index3005 = err.rfind('(error code: 3005') # (no user mapping) - INFN + if err_redir_index >= 0 and (err_index3010 >= 0 or err_index3005 >= 0): + errtxt = '' + else: + if(ret > 0): + errtxt = "client-side error - exit code "+str(ret)+"\n" + err_index = err.rfind('Last server error') + if err_index >= 0: + err_end_index=err.find("\n",err_index) + errtxt = errtxt + err[err_index:err_end_index] + except Exception,e: + errtext = errtxt + "Exception: "+str(e) + dev_null.close() + return (errtxt,out,err,elapsed) +def test_redirector(dicci, servicename, redirector, file_below=None, file_above=None, extra_notes=""): + servicename=servicename.upper() + notes_text = "Redirector: "+redirector + availability = 'Available' + availinfo = '' + c = '' + if 'status' in dicci and dicci['status'] == 'unavailable': + pass + elif file_below == None and file_above == None: + availability = 'Unavialable' + c = 'Non-existing File Above and File Below.' + dicci['xrdcp_below_time'] = 0 + dicci['xrdcp_below_time'] = 0 + else: + if (file_below): + notes_text = notes_text + "File below: " + file_below + (err_below,dump_below,elapsed_below) = xrdcp_test(redirector, file_below) + if err_below: + availability = 'Degraded' + #availinfo=availinfo+" Error below redirector "+err_below + dump_sane = re.sub('---*','__',dump_below) + c = c+"Error for file BELOW: "+err_below+". Dumpsane: "+dump_sane+ '.' + dicci['xrdcp_below_time'] = 0 + else: + #availinfo=availinfo+" File below: OK " + dicci['xrdcp_below_time'] = elapsed_below + else: + c = "Error for file BELOW: Non-existing File Below. " + dicci['xrdcp_below_time'] = 0 + if(file_above): + notes_text = notes_text + "File elsewhere: " + file_above + (err_above,dump_above,elapsed_above) = xrdcp_test(redirector, file_above) + if err_above : + availability = 'Degraded' + #availinfo=availinfo+" Error above redirector "+err_above + dump_sane = re.sub('---*','__',dump_above) + c = c+"Error for file ABOVE: "+err_above+". Dumpsane: "+dump_sane+'.' + dicci['xrdcp_above_time'] = 0 + else: + #availinfo = availinfo+" File above: OK " + dicci['xrdcp_above_time'] = elapsed_above + else: + c = c + "Error for file ABOVE: Non-existing File Above." + dicci['xrdcp_above_time'] = 0 + #availinfo = availinfo + " " + notes_text + dicci['status']= str(availability) + if c == '': + c = 'N/A' + c = c.replace("\n", "") + c = c.replace("\r", "") + dicci ['Comment'] = c + with open(html_dir +'KIBANA_PROBES.json', 'a') as f: + json.dump(dicci, f) + f.write('\n') +def main(): + debug = 0 + atexit.register(clear_lock) + if len(sys.argv) > 1: + if sys.argv[1] == '-d': + debug=1 + if not try_lock(): + sys.exit(1) + if not os.path.exists(html_dir): + os.makedirs(html_dir) + env_setup() + timeout_sec = 10 * 60 # limit overall runtime to 10min + signal.signal(signal.SIGALRM, alarm_handler) + ATLASLINK="%BR%Monitoring:%BR%\n http://atl-prod07.slac.stanford.edu:8080/display?page=xrd_report/aggregated/total_xrootd_lgn %BR%\n http://dashb-atlas-xrootd-transfers.cern.ch/ui %BR%\nhttp://dashb-atlas-ssb.cern.ch/dashboard/request.py/siteview#currentView=FAX+redirectors&highlight=false %BR%\n" + CMSLINK="%BR%Monitoring:%BR%\n http://xrootd.t2.ucsd.edu/dashboard/ %BR%\n http://dashb-cms-xrootd-transfers.cern.ch/ui %BR%\n" + FILEABOVE="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + FILEBELOW="/store/mc/SAM/GenericTTbar/AODSIM/CMSSW_9_2_6_91X_mcRun1_realistic_v2-v1/00000/A64CCCF2-5C76-E711-B359-0CC47A78A3F8.root" + services = { + "XRDFED_CMS-GLOBAL01-NEW":{'redirector':'cms-xrd-global01.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL02-NEW":{'redirector':'cms-xrd-global02.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-FNAL":{'redirector':'cmsxrootd2.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US-UNL":{'redirector':'xrootd.unl.edu:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-BARI":{'redirector':'xrootd.ba.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-LLR":{'redirector':'llrxrd-redir.in2p3.fr:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-PISA":{'redirector':'xrootd-redic.pi.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-GLOBAL":{'redirector':'cms-xrd-global.cern.ch:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-US":{'redirector':'cmsxrootd.fnal.gov:1094', + 'file_below': FILEABOVE, + 'file_above': FILEBELOW, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU":{'redirector':'xrootd-cms.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-EU-IPv6":{ 'redirector':'xrootd-cms-redir-01.cr.cnaf.infn.it:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT":{'redirector':'cms-xrd-transit.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT01":{'redirector':'vocms031.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + "XRDFED_CMS-TRANSIT02":{'redirector':'vocms032.cern.ch:1094', + 'file_below': FILEBELOW, + 'file_above': FILEABOVE, + 'extra_notes':CMSLINK}, + } + signal.alarm(timeout_sec) + try: + diccionaries = [] + for xrd in services: + services[xrd].update(servicename=xrd) + servicename = xrd + dicci = prepare_dictionary(servicename, services[xrd]['redirector']) + diccionaries.append(dicci) + for dicci in diccionaries: + service = dicci['service'] + argus = services[service] + argus['dicci'] = dicci + if debug: + test_redirector(** services[xrd]) + else: + t = threading.Thread(target=test_redirector, kwargs = argus) # read: "run a thread with the test function and all the parameters above as arguments" + t.start() + #t.join() + #os.system('source ~/single_quotes.sh') + except Alarm: + print "ERROR: caught overall timeout after "+str(timeout_sec)+"s\n" + clear_lock() + sys.exit(2) + signal.alarm(0) +if __name__ == '__main__': + for file in os.listdir(html_dir): + if file == 'KIBANA_PROBES.json': + os.remove(html_dir+'KIBANA_PROBES.json') + break + main() diff --git a/AAAOps/XfedKibana_JSON/kibana_probe.sh b/AAAOps/XfedKibana_JSON/kibana_probe.sh new file mode 100644 index 0000000..39c1407 --- /dev/null +++ b/AAAOps/XfedKibana_JSON/kibana_probe.sh @@ -0,0 +1,2 @@ +python ./XRDFED-kibana-probe_JSON.py +source ./single_quotes.sh diff --git a/AAAOps/XfedKibana_JSON/single_quotes.sh b/AAAOps/XfedKibana_JSON/single_quotes.sh new file mode 100644 index 0000000..8afd322 --- /dev/null +++ b/AAAOps/XfedKibana_JSON/single_quotes.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sed -i "y/\"/'/" /var/www/html/aaa-probe/KIBANA_PROBES.json diff --git a/AAAOps/send_aaa_matrics/aaa_federation.py b/AAAOps/send_aaa_matrics/aaa_federation.py new file mode 100755 index 0000000..13c781c --- /dev/null +++ b/AAAOps/send_aaa_matrics/aaa_federation.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +import os +import sys +import subprocess +import urllib +import json +from datetime import datetime +import collections +import stomp +import argparse +from CMSMonitoring.StompAMQ import StompAMQ +import uuid +from itertools import islice +import hashlib +import stomp +from uuid import uuid4 +from urllib.request import urlopen +import time + +print("stomp version", stomp.__version__) + + +class OptionParser(): + def __init__(self): + "User based option parser" + self.parser = argparse.ArgumentParser() + msg = "Send aaa subscription json file via StompAMQ to a broker, provide broker credentials in JSON file" + self.parser.add_argument("--amq", action="store", + dest="amq", default="", help=msg, required=True) + + +class MyListener(stomp.ConnectionListener): + def on_error(self, headers, message): + print('received an error "%s"' % message) + + def on_message(self, headers, message): + print('received a message "%s"' % message) + + +def credentials(fname=None): + "Read credentials from MONIT_BROKER environment" + if not fname: + fname = os.environ.get('MONIT_BROKER', '') + if not os.path.isfile(fname): + raise Exception( + "Unable to locate MONIT credentials, please setup MONIT_BROKER") + return {} + with open(fname, 'r') as istream: + data = json.load(istream) + return data + + +def parseFederationJson(): + ''' + modifies federation.json with tags and + Return json object to feed cern metris + ''' + url = 'http://vocms039.cern.ch/aaa-fedinfo/federations.json' + result = json.load(urlopen(url)) + fed_data = createFlattenJson(result) + f = open("fed.json", "r") + for x in f: + data = json.loads(x) + yield data + +def createFlattenJson(json_file): + ''' + flattens the federation.json file + ''' + for k,v in json_file.items(): + if ("prod" in k): + statusCode = 1 + stateName = "Production Federation" + elif ("trans" in k): + statusCode = 2 + stateName = "Transitional Federation" + else: + statusCode = 3 + stateName = "not at any federation" + for site in v: + my_json = ('{"siteName" : "%s", "state" : "%s", "statusCode" : %s, "stateName" : "%s"}' %(site, k, statusCode, stateName)) + with open("fed.json","a") as f: + f.write("%s\n" %my_json) + +if __name__ == "__main__": + optmgr = OptionParser() + opts = optmgr.parser.parse_args() + payload = parseFederationJson() + if opts.amq: + creds = credentials(opts.amq) + username = creds['username'] + password = creds['password'] + topic = creds['topic'] + host, port = creds['host_and_ports'].split(':') + port = int(port) + producer = creds['producer'] + topic = creds['topic'] + hosts = [(host, port)] + if creds and StompAMQ: + amq = StompAMQ(username, password, producer, topic, key=None, + cert=None, validation_schema=None, host_and_ports=[(host, port)]) + eod = False + wait_seconds = 10 + while not eod: + messages = [] + for d in payload: + notif,_,_ = amq.make_notification(d, "aaa_federations_document", dataSubfield=None) + messages.append(notif) + if messages: + print(messages) + amq.send(messages) + time.sleep(wait_seconds) + else: + eod = True diff --git a/AAAOps/send_aaa_matrics/aaa_federation.sh b/AAAOps/send_aaa_matrics/aaa_federation.sh new file mode 100755 index 0000000..8ab5812 --- /dev/null +++ b/AAAOps/send_aaa_matrics/aaa_federation.sh @@ -0,0 +1,9 @@ +#!/bin/bash + + +set -x +export PYTHONPATH=$PYTHONPATH:/opt/TransferTeam/AAAOps/send_aaa_matrics/CMSMonitoring/src/python/ +export PYTHONPATH=$PYTHONPATH:/opt/TransferTeam/AAAOps/send_aaa_matrics/stomp.py/stomp/ +#source /cvmfs/sft.cern.ch/lcg/views/LCG_96python3/x86_64-centos7-gcc8-opt/setup.sh +rm fed.json +python3 /opt/TransferTeam/AAAOps/send_aaa_matrics/aaa_federation.py --amq /opt/TransferTeam/AAAOps/send_aaa_matrics/credentials.json diff --git a/AAAOps/tools/timestamp.sh b/AAAOps/tools/timestamp.sh old mode 100644 new mode 100755 index ab57df7..c3c0590 --- a/AAAOps/tools/timestamp.sh +++ b/AAAOps/tools/timestamp.sh @@ -1,4 +1,5 @@ #!/bin/bash + while read x; do echo -n `date +%d/%m/%Y\ %H:%M:%S`; echo -n " "; diff --git a/consistency_check/spark/README.md b/consistency_check/spark/README.md new file mode 100644 index 0000000..7eb1392 --- /dev/null +++ b/consistency_check/spark/README.md @@ -0,0 +1,24 @@ + +# Spark code to run consistency between DBS and Phedex databases + + + +## Basic lxplus7 setup + + +login to lxplus7.cern.ch + + +**sh setup.sh** + + + +## run dbs and phedex consistency check on spark: + + + +## It returns the list of file which are invalidated by the unified for bad workflows +Example : +--timestamp argument for checking the list for last n number of days + +python filemismatch.py --past_days 7 diff --git a/consistency_check/spark/dbs_phedex_consistency.py b/consistency_check/spark/dbs_phedex_consistency.py new file mode 100644 index 0000000..269357a --- /dev/null +++ b/consistency_check/spark/dbs_phedex_consistency.py @@ -0,0 +1,112 @@ +#!/usr/bin/env spark-submit +from __future__ import print_function +import argparse +import json +import os + +from pyspark import SparkConf, SparkContext, StorageLevel +from pyspark.sql import SparkSession, Column +from pyspark.sql.functions import col +import pyspark.sql.functions as fn +import pyspark.sql.types as types +from pyspark.sql import DataFrame +from datetime import date , timedelta +# stolen from CMSSpark +import schemas + +class OptionParser(): + def __init__(self): + "option Parser" + self.parser = argparse.ArgumentParser(prog='consistency') + msg= "Output path in HDFS for result" + self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) + +class run_consistency(object): + def __init__(self, out): + self.out = out + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + self.spark = SparkSession(sc) + avroreader = self.spark.read.format("com.databricks.spark.avro") + csvreader = self.spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode","FAILFAST") + ## check if phedex_path exist or not on hdfs area and you can directly assign it + phedex_path = "/project/awg/cms/phedex/block-replicas-snapshots/csv/time=" + str((date.today() - timedelta(days=2)).strftime("%Y-%m-%d")) + "_*/part-m-00000" + self.phedex_block_replicas = csvreader.schema(schemas.schema_phedex()).load(phedex_path) + self.dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/FILES/part-m-00000") + self.dbs_blocks = csvreader.schema(schemas.schema_blocks()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/BLOCKS/part-m-00000") + self.dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/new/DATASETS/part-m-00000") + + def invalid_dbs_present_phedex(self): + ''' + + Returns a dataframe with datasets which have "INVALID" status in DBS and are "PRESENT" in phedex + + :func: run_consistency.invalid_dbs_present_phedex() + + for reference dbs d_dataset_access_type_id: + 1 : valid + 2 : invalid + 42 : Deprecated + 41 : Production + 81 : Deleted + ''' + invalid_dbs_present_phedex = (self.dbs_datasets + .filter(col('d_dataset_access_type_id')=='2') + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('dataset_name').isNotNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','d_dataset','d_last_modified_by') # you can select more columns for detail info + .distinct()) + + invalid_dbs_present_phedex.groupby("input_campaign").agg((fn.count(fn.col("d_dataset")))).show() + return invalid_dbs_present_phedex.select("d_dataset") + + def deleted_dbs_present_phedex(self): + ''' + Returns a dataframe with datasets which have "DELETED" status in DBS and "PRESENT" status in phedex + + :func: run_consistency.deleted_dbs_present_phedex() + + ''' + out = args.out_path + "/deleted_dbs_present_phedex" + deleted_dbs_present_phedex = (self.dbs_datasets + .filter(col('d_dataset_access_type_id')=='81') + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('dataset_name').isNotNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','d_dataset') # you can select more columns for detail info + .distinct()) + + deleted_dbs_present_phedex.groupby("input_campaign").agg(fn.count(fn.col("d_dataset"))).show() + return deleted_dbs_present_phedex.select("d_dataset") + + def valid_dbs_missing_phedex(self): + ''' + Returns a dataframe with block name which have "VALID" status in DBS and are "MISSING" in phedex + + :func: run_consistency.valid_dbs_missing_phedex() + + ''' + out = args.out_path + "/valid_dbs_missing_phedex" + valid_dbs_missing_phedex = (self.dbs_datasets + .filter(col('d_dataset_access_type_id')=='1') + .join(self.dbs_blocks,col('d_dataset_id')==col('b_dataset_id')) + .join(self.phedex_block_replicas,col('d_dataset')==col('dataset_name')) + .filter(col('block_name').isNotNull() & col('node_id').isNull()) + .withColumn('input_campaign', fn.regexp_extract(col('d_dataset'), "^/[^/]*/((?:HI|PA|PN|XeXe|)Run201\d\w-[^-]+|CMSSW_\d+|[^-]+)[^/]*/", 1)) + .select('input_campaign','block_name') + .distinct()) + + print("Please check the below list of datasets with phedex api calls , Its possible you encounter discrepency due to the time difference of dataset injection in both databases and snapshot stored on HDFS") + valid_dbs_missing_phedex.groupby("input_campaign").agg(fn.count(fn.col("block_name"))).show() + return valid_dbs_missing_phedex.select("block_name") + + + +if __name__ == '__main__': + optmgr = OptionParser() + args = optmgr.parser.parse_args() + con = run_consistency(args.out_path) + invalid_dbs_present_phedex = con.deleted_dbs_present_phedex() diff --git a/consistency_check/spark/filemismatch.py b/consistency_check/spark/filemismatch.py new file mode 100644 index 0000000..5ffaa49 --- /dev/null +++ b/consistency_check/spark/filemismatch.py @@ -0,0 +1,68 @@ +#!/usr/bin/env spark-submit +from __future__ import print_function +import argparse +import json +import os + +from pyspark import SparkConf, SparkContext, StorageLevel +from pyspark.sql import SparkSession, Column +from pyspark.sql.functions import col +import pyspark.sql.functions as fn +import pyspark.sql.types as types +from pyspark.sql import DataFrame +import datetime +import time + + +# stolen from CMSSpark +import schemas + +class OptionParser(): + """Dataset access types: + 81,DELETED + 42,DEPRECATED + 2,INVALID + 41,PRODUCTION + 1,VALID + """ + def __init__(self): + "option Parser" + self.parser = argparse.ArgumentParser(prog='consistency') + msg = "Files mismatch since last n days : default in past 7 days" + self.parser.add_argument("--past_days",action="store",dest="days",default=7,help=msg,type=int) + msg= "Output path in HDFS for result" + self.parser.add_argument("--out", action="store", dest="out_path",default=None,help=msg,required=True) + +def fileMismatch(args): + conf = SparkConf().setMaster("yarn").setAppName("CMS Working Set") + sc = SparkContext(conf=conf) + spark = SparkSession(sc) + print("Initiated spark session on yarn, web URL: http://ithdp1101.cern.ch:8088/proxy/%s" % sc.applicationId) + + avroreader = spark.read.format("com.databricks.spark.avro") + csvreader = spark.read.format("com.databricks.spark.csv").option("nullValue","null").option("mode", "FAILFAST") + dbs_files = csvreader.schema(schemas.schema_files()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/FILES/part-m-00000") + dbs_datasets = csvreader.schema(schemas.schema_datasets()).load("/project/awg/cms/CMS_DBS3_PROD_GLOBAL/current/DATASETS/part-m-00000") + + current = time.time() + past_n_days = args.days + delta_t = current - past_n_days*60*60*24 + delta_t_str = str(delta_t) + delta_t = delta_t_str[:10] + + if args.out_path: + mismatch_df = (dbs_files + .filter(col('f_is_file_valid') == '0') + .filter(col('f_last_modification_date') > delta_t) + .join(dbs_datasets,col('f_dataset_id') == col('d_dataset_id')) + .filter((col('d_dataset_access_type_id') == '1') | (col('d_dataset_access_type_id') == '41')) + .filter(col('f_logical_file_name').isNotNull()) + .where(~(dbs_files.f_last_modified_by.contains('dmielaik') | dbs_files.f_last_modified_by.contains('ogarzonm'))) + .select('d_dataset','f_last_modified_by','f_logical_file_name') + .distinct()) + mismatch_df.select('f_logical_file_name','f_last_modified_by').repartition(1).write.format("com.databricks.spark.csv").option("header", "true").save(args.out_path) + mismatch_df.groupby('d_dataset').agg(fn.count(fn.col("f_logical_file_name")).alias('extra_lfn_phedex')).show() +if __name__ == '__main__': + optmgr = OptionParser() + args = optmgr.parser.parse_args() + fileMismatch(args) diff --git a/consistency_check/spark/schemas.py b/consistency_check/spark/schemas.py new file mode 100644 index 0000000..c624c62 --- /dev/null +++ b/consistency_check/spark/schemas.py @@ -0,0 +1,523 @@ + +#!/usr/bin/env python +#-*- coding: utf-8 -*- +#pylint: disable= +# Author: Valentin Kuznetsov +""" +Schema module for DBS/PhEDEx/AAA/EOS/CMSSW/JobMonitoring meta-data on HDFS +""" + +# system modules +import os +import sys +import time +import json + +# spark modules +from pyspark.sql.types import DoubleType, IntegerType, StructType, StructField, StringType, BooleanType, LongType + +def schema_processing_eras(): + """ + DBS PROCESSING_ERAS table schema + PROCESSING_ERA_ID NOT NULL NUMBER(38) + PROCESSING_ERA_NAME NOT NULL VARCHAR2(120) + CREATION_DATE NOT NULL INTEGER + CREATE_BY NOT NULL VARCHAR2(500) + DESCRIPTION NOT NULL VARCHAR2(40) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("processing_era_id", IntegerType(), True), + StructField("processing_version", StringType(), True), + StructField("creation_date", IntegerType(), True), + StructField("create_by", StringType(), True), + StructField("description", StringType(), True) + ]) + +def schema_acquisition_eras(): + """ + DBS ACQUISITION_ERAS table schema + ACQUISITION_ERA_ID NOT NULL NUMBER(38) + ACQUISITION_ERA_NAME NOT NULL VARCHAR2(120) + START_DATE NOT NULL INTEGER + END_DATE NOT NULL INTEGER + CREATION_DATE NOT NULL INTEGER + CREATE_BY NOT NULL VARCHAR2(500) + DESCRIPTION NOT NULL VARCHAR2(40) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("acquisition_era_id", IntegerType(), True), + StructField("acquisition_era_name", StringType(), True), + StructField("start_date", IntegerType(), True), + StructField("end_date", IntegerType(), True), + StructField("creation_date", IntegerType(), True), + StructField("create_by", StringType(), True), + StructField("description", StringType(), True) + ]) + +def schema_dataset_access_types(): + """ + DBS DATASET_ACCESS_TYPES table schema + DATASET_ACCESS_TYPE_ID NOT NULL NUMBER(38) + DATASET_ACCESS_TYPE NOT NULL VARCHAR2(100) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("dataset_access_type_id", IntegerType(), True), + StructField("dataset_access_type", StringType(), True) + ]) + +def schema_data_tiers(): + """ + DBS DATA_TIERS table schema + DATA_TIER_ID NOT NULL NUMBER(38) + DATA_TIER_NAME NOT NULL VARCHAR2(100) + CREATION_DATE NOT NULL NUMBER(38) + CREATE_BY NOT NULL VARCHAR2(100) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("data_tier_id", IntegerType(), True), + StructField("data_tier_name", StringType(), True), + StructField("data_tier_creation_date", DoubleType(), True), + StructField("data_tier_create_by", StringType(), True) + ]) + +def schema_datasets(): + """ + DBS DATASETS table schema + DATASET_ID NOT NULL NUMBER(38) + DATASET NOT NULL VARCHAR2(700) + IS_DATASET_VALID NOT NULL NUMBER(38) + PRIMARY_DS_ID NOT NULL NUMBER(38) + PROCESSED_DS_ID NOT NULL NUMBER(38) + DATA_TIER_ID NOT NULL NUMBER(38) + DATASET_ACCESS_TYPE_ID NOT NULL NUMBER(38) + ACQUISITION_ERA_ID NUMBER(38) + PROCESSING_ERA_ID NUMBER(38) + PHYSICS_GROUP_ID NUMBER(38) + XTCROSSSECTION FLOAT(126) + PREP_ID VARCHAR2(256) + CREATION_DATE NUMBER(38) + CREATE_BY VARCHAR2(500) + LAST_MODIFICATION_DATE NUMBER(38) + LAST_MODIFIED_BY VARCHAR2(500) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("d_dataset_id", IntegerType(), True), + StructField("d_dataset", StringType(), True), + StructField("d_is_dataset_valid", IntegerType(), True), + StructField("d_primary_ds_id", IntegerType(), True), + StructField("d_processed_ds_id", IntegerType(), True), + StructField("d_data_tier_id", IntegerType(), True), + StructField("d_dataset_access_type_id", IntegerType(), True), + StructField("d_acquisition_era_id", IntegerType(), True), + StructField("d_processing_era_id", IntegerType(), True), + StructField("d_physics_group_id", IntegerType(), True), + StructField("d_xtcrosssection", DoubleType(), True), + StructField("d_prep_id", StringType(), True), + StructField("d_creation_date", DoubleType(), True), + StructField("d_create_by", StringType(), True), + StructField("d_last_modification_date", DoubleType(), True), + StructField("d_last_modified_by", StringType(), True) + ]) + +def schema_blocks(): + """ + DBS BLOCKS table schema + BLOCK_ID NOT NULL NUMBER(38) + BLOCK_NAME NOT NULL VARCHAR2(500) + DATASET_ID NOT NULL NUMBER(38) + OPEN_FOR_WRITING NOT NULL NUMBER(38) + ORIGIN_SITE_NAME NOT NULL VARCHAR2(100) + BLOCK_SIZE NUMBER(38) + FILE_COUNT NUMBER(38) + CREATION_DATE NUMBER(38) + CREATE_BY VARCHAR2(500) + LAST_MODIFICATION_DATE NUMBER(38) + LAST_MODIFIED_BY VARCHAR2(500) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("b_block_id", IntegerType(), True), + StructField("b_block_name", StringType(), True), + StructField("b_dataset_id", IntegerType(), True), + StructField("b_open_for_writing", IntegerType(), True), + StructField("b_origin_site_name", StringType(), True), + StructField("b_block_size", DoubleType(), True), + StructField("b_file_count", IntegerType(), True), + StructField("b_creation_date", DoubleType(), True), + StructField("b_create_by", StringType(), True), + StructField("b_last_modification_date", DoubleType(), True), + StructField("b_last_modified_by", StringType(), True) + ]) + +def schema_files(): + """ + DBS FILES table schema + FILE_ID NOT NULL NUMBER(38) + LOGICAL_FILE_NAME NOT NULL VARCHAR2(500) + IS_FILE_VALID NOT NULL NUMBER(38) + DATASET_ID NOT NULL NUMBER(38) + BLOCK_ID NOT NULL NUMBER(38) + FILE_TYPE_ID NOT NULL NUMBER(38) + CHECK_SUM NOT NULL VARCHAR2(100) + EVENT_COUNT NOT NULL NUMBER(38) + FILE_SIZE NOT NULL NUMBER(38) + BRANCH_HASH_ID NUMBER(38) + ADLER32 VARCHAR2(100) + MD5 VARCHAR2(100) + AUTO_CROSS_SECTION FLOAT(126) + CREATION_DATE NUMBER(38) + CREATE_BY VARCHAR2(500) + LAST_MODIFICATION_DATE NUMBER(38) + LAST_MODIFIED_BY VARCHAR2(500) + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("f_file_id", IntegerType(), True), + StructField("f_logical_file_name", StringType(), True), + StructField("f_is_file_valid", IntegerType(), True), + StructField("f_dataset_id", IntegerType(), True), + StructField("f_block_id", IntegerType(), True), + StructField("f_file_type_id", IntegerType(), True), + StructField("f_check_sum", StringType(), True), + StructField("f_event_count", IntegerType(), True), + StructField("f_file_size", DoubleType(), True), + StructField("f_branch_hash_id", IntegerType(), True), + StructField("f_adler32", StringType(), True), + StructField("f_md5", StringType(), True), + StructField("f_auto_cross_section", DoubleType(), True), + StructField("f_creation_date", DoubleType(), True), + StructField("f_create_by", StringType(), True), + StructField("f_last_modification_date", DoubleType(), True), + StructField("f_last_modified_by", StringType(), True) + ]) + +def schema_mod_configs(): + """ + DBS DATASET_OUTPUT_MOD_CONFIGS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("mc_ds_output_mod_config_id", IntegerType(), True), + StructField("mc_dataset_id", IntegerType(), True), + StructField("mc_output_mod_config_id", IntegerType(), True) + ]) + +def schema_out_configs(): + """ + DBS OUTPUT_MODULE_CONFIGS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("oc_output_mod_config_id", IntegerType(), True), + StructField("oc_app_exec_id", IntegerType(), True), + StructField("oc_release_version_id", IntegerType(), True), + StructField("oc_parameter_set_hash_id", IntegerType(), True), + StructField("oc_output_module_label", StringType(), True), + StructField("oc_global_tag", StringType(), True), + StructField("oc_scenario", StringType(), True), + StructField("oc_creation_date", IntegerType(), True), + StructField("oc_create_by", StringType(), True) + ]) + +def schema_rel_versions(): + """ + DBS RELEASE_VERSIONS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("r_release_version_id", IntegerType(), True), + StructField("r_release_version", StringType(), True) + ]) + +def schema_file_lumis(): + """DBS FILE_LUMIS table schema + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("fl_run_num", IntegerType(), True), + StructField("fl_lumi_section_num", StringType(), True), + StructField("fl_file_id", IntegerType(), True) + ]) + +def schema_phedex_summary(): + """PhEDEx summary table schema + site,dataset,size,date,replica_date + :returns: StructType consisting StructField array + """ + return StructType([ + StructField("date", LongType(), True), + StructField("site", StringType(), True), + StructField("dataset", StringType(), True), + StructField("size", LongType(), True), + StructField("replica_date", LongType(), True) + ]) + +def schema_phedex(): + """ + PhEDEx schema on HDFS + :returns: StructType consisting StructField array + """ + return StructType([StructField("now_sec", DoubleType(), True), + StructField("dataset_name", StringType(), True), + StructField("dataset_id", IntegerType(), True), + StructField("dataset_is_open", StringType(), True), + StructField("dataset_time_create", DoubleType(), True), + StructField("dataset_time_update", DoubleType(), True), + StructField("block_name", StringType(), True), + StructField("block_id", IntegerType(), True), + StructField("block_files", IntegerType(), True), + StructField("block_bytes", LongType(), True), + StructField("block_is_open", StringType(), True), + StructField("block_time_create", DoubleType(), True), + StructField("block_time_update", DoubleType(), True), + StructField("node_name", StringType(), True), + StructField("node_id", IntegerType(), True), + StructField("br_is_active", StringType(), True), + StructField("br_src_files", LongType(), True), + StructField("br_src_bytes", LongType(), True), + StructField("br_dest_files", LongType(), True), + StructField("br_dest_bytes", LongType(), True), + StructField("br_node_files", LongType(), True), + StructField("br_node_bytes", LongType(), True), + StructField("br_xfer_files", LongType(), True), + StructField("br_xfer_bytes", LongType(), True), + StructField("br_is_custodial", StringType(), True), + StructField("br_user_group_id", IntegerType(), True), + StructField("replica_time_create", DoubleType(), True), + StructField("replica_time_updater", DoubleType(), True)]) + +def schema_dbs_phedex(): + """ + Schema for DBS+PhEDEx aggregation, see dbs_phedex.py and adjust accordingly. + dataset_name,evts,size,date,dataset_access_type,acquisition_era_name,r_release_version,node_name,pbr_size,dataset_is_open,max_replica_time + "/14TeV_T1T1_2BC_350_100_MadGraph/Summer12-UpgrdStdGeom_DESIGN42_V17-v1/GEN",73528,4.4569181E7,1.343676821E9,VALID,DBS2_UNKNOWN_ACQUISION_ERA,CMSSW_4_2_8_SLHCstd2_patch2,T1_US_FNAL_Buffer,4.4569181E7,y,1.361368512E9 + """ + return StructType([ + StructField("dataset_name", StringType(), True), + StructField("evts", IntegerType(), True), + StructField("size", DoubleType(), True), + StructField("date", DoubleType(), True), + StructField("dataset_access_type", StringType(), True), + StructField("acquisition_era_name", StringType(), True), + StructField("r_release_version", StringType(), True), + StructField("node_name", StringType(), True), + StructField("pbr_size", DoubleType(), True), + StructField("dataset_is_open", StringType(), True), + StructField("max_replica_time", DoubleType(), True) + ]) + +def schema_cmssw(): + """Schema for CMSSW record + {"UNIQUE_ID":{"string":"08F8DD3A-0FFE-E611-B710-BC305B3909F1-1"},"FILE_LFN":{"string":"/store/data/Run2016F/JetHT/AOD/23Sep2016-v1/70000/D2B97318-A186-E611-A1EA-F8BC123BBE3C.root"},"FILE_SIZE":{"string":"3865077537"},"CLIENT_DOMAIN":{"string":"in2p3.fr"},"CLIENT_HOST":{"string":"sbgwn141"},"SERVER_DOMAIN":{"string":"in2p3.fr"},"SERVER_HOST":{"string":"sbgse20"},"SITE_NAME":{"string":"T2_FR_IPHC"},"READ_BYTES_AT_CLOSE":{"string":"438385807"},"READ_BYTES":{"string":"438385807"},"READ_SINGLE_BYTES":{"string":"8913451"},"READ_SINGLE_OPERATIONS":{"string":"19"},"READ_SINGLE_AVERAGE":{"string":"469129"},"READ_SINGLE_SIGMA":{"string":"1956390"},"READ_VECTOR_BYTES":{"string":"429472356"},"READ_VECTOR_OPERATIONS":{"string":"58"},"READ_VECTOR_AVERAGE":{"string":"7404700"},"READ_VECTOR_SIGMA":{"string":"6672770"},"READ_VECTOR_COUNT_AVERAGE":{"string":"37.4138"},"READ_VECTOR_COUNT_SIGMA":{"string":"35.242"},"FALLBACK":{"string":"-"},"USER_DN":{"string":"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=vmariani/CN=751637/CN=Valentina Mariani/CN=1516526926/CN=481221408/CN=1301887600/CN=1846615186/CN=2041527197"},"APP_INFO":{"string":"3809_https://glidein.cern.ch/3809/170228:163214:vmariani:crab:JetHT:Run2016F:DSm:4b_0"},"START_TIME":{"string":"1488325657"},"END_TIME":{"string":"1488326400"},"START_DATE":{"long":1488322057000},"END_DATE":{"long":1488322800000},"INSERT_DATE":{"long":1488323999000}} + """ + return StructType([ + StructField("UNIQUE_ID", StringType(), True), + StructField("FILE_LFN", StringType(), True), + StructField("FILE_SIZE", StringType(), True), + StructField("CLIENT_DOMAIN", StringType(), True), + StructField("CLIENT_HOST", StringType(), True), + StructField("SERVER_DOMAIN", StringType(), True), + StructField("SERVER_HOST", StringType(), True), + StructField("SITE_NAME", StringType(), True), + StructField("READ_BYTES_AT_CLOSE", StringType(), True), + StructField("READ_BYTES", StringType(), True), + StructField("READ_SINGLE_BYTES", StringType(), True), + StructField("READ_SINGLE_OPERATIONS", StringType(), True), + StructField("READ_SINGLE_AVERAGE", StringType(), True), + StructField("READ_SINGLE_SIGMA", StringType(), True), + StructField("READ_VECTOR_BYTES", StringType(), True), + StructField("READ_VECTOR_OPERATIONS", StringType(), True), + StructField("READ_VECTOR_AVERAGE", StringType(), True), + StructField("READ_VECTOR_SIGMA", StringType(), True), + StructField("READ_VECTOR_COUNT_AVERAGE", StringType(), True), + StructField("READ_VECTOR_COUNT_SIGMA", StringType(), True), + StructField("USER_DN", StringType(), True), + StructField("APP_INFO", StringType(), True), + StructField("START_TIME", StringType(), True), + StructField("END_TIME", StringType(), True), + StructField("START_DATE", LongType(), True), + StructField("END_DATE", LongType(), True), + StructField("INSERT_DATE", LongType(), True) + ]) + +def schema_jm(): + """Schema for JobMonitoring record + {"JobId":{"string":"1672451388"},"FileName":{"string":"//store/mc/PhaseIIFall16GS82/QCD_Flat_Pt-15to7000_TuneCUETP8M1_14TeV_pythia8/GEN-SIM/90X_upgrade2023_realistic_v1-v1/110000/92A9E11F-C0F1-E611-9A55-001E67E6F8AF.root"},"IsParentFile":{"string":"0"},"ProtocolUsed":{"string":"Remote"},"SuccessFlag":{"string":"1"},"FileType":{"string":"EDM"},"LumiRanges":{"string":"unknown"},"StrippedFiles":{"string":"0"},"BlockId":{"string":"602064"},"StrippedBlocks":{"string":"0"},"BlockName":{"string":"Dummy"},"InputCollection":{"string":"DoesNotApply"},"Application":{"string":"CMSSW"},"Type":{"string":"reprocessing"},"SubmissionTool":{"string":"wmagent"},"InputSE":null,"TargetCE":null,"SiteName":{"string":"T0_CH_CERN"},"SchedulerName":{"string":"PYCONDOR"},"JobMonitorId":{"string":"unknown"},"TaskJobId":{"string":"1566463230"},"SchedulerJobIdV2":{"string":"664eef36-f1c3-11e6-88b9-02163e0184a6-367_0"},"TaskId":{"string":"35076445"},"TaskMonitorId":{"string":"wmagent_pdmvserv_task_SMP-PhaseIIFall16GS82-00005__v1_T_170213_041344_640"},"JobExecExitCode":{"string":"0"},"JobExecExitTimeStamp":{"long":1488375506000},"StartedRunningTimeStamp":{"long":1488374686000},"FinishedTimeStamp":{"long":1488375506000},"WrapWC":{"string":"820"},"WrapCPU":{"string":"1694.3"},"ExeCPU":{"string":"0"},"UserId":{"string":"124370"},"GridName":{"string":"Alan Malta Rodrigues"}} + """ + return StructType([ + StructField("JobId", StringType(), True), + StructField("FileName", StringType(), True), + StructField("IsParentFile", StringType(), True), + StructField("ProtocolUsed", StringType(), True), + StructField("SuccessFlag", StringType(), True), + StructField("FileType", StringType(), True), + StructField("LumiRanges", StringType(), True), + StructField("StrippedFiles", StringType(), True), + StructField("BlockId", StringType(), True), + StructField("StrippedBlocks", StringType(), True), + StructField("BlockName", StringType(), True), + StructField("InputCollection", StringType(), True), + StructField("Application", StringType(), True), + StructField("ApplicationVersion", StringType(), True), + StructField("Type", StringType(), True), + StructField("GenericType", StringType(), True), + StructField("NewGenericType", StringType(), True), + StructField("NewType", StringType(), True), + StructField("SubmissionTool", StringType(), True), + StructField("InputSE", StringType(), True), + StructField("TargetCE", StringType(), True), + StructField("SiteName", StringType(), True), + StructField("SchedulerName", StringType(), True), + StructField("JobMonitorId", StringType(), True), + StructField("TaskJobId", StringType(), True), + StructField("SchedulerJobIdV2", StringType(), True), + StructField("TaskId", StringType(), True), + StructField("TaskMonitorId", StringType(), True), + StructField("NEventsPerJob", StringType(), True), + StructField("NTaskSteps", StringType(), True), + StructField("JobExecExitCode", StringType(), True), + StructField("JobExecExitTimeStamp", LongType(), True), + StructField("StartedRunningTimeStamp", LongType(), True), + StructField("FinishedTimeStamp", LongType(), True), + StructField("WrapWC", StringType(), True), + StructField("WrapCPU", StringType(), True), + StructField("ExeCPU", StringType(), True), + StructField("NCores", StringType(), True), + StructField("NEvProc", StringType(), True), + StructField("NEvReq", StringType(), True), + StructField("WNHostName", StringType(), True), + StructField("JobType", StringType(), True), + StructField("UserId", StringType(), True), + StructField("GridName", StringType(), True) + ]) + +def schema_asodb(): + """ + ASO table schema + Map for values of tm_transfer_state and tm_publication_state in + https://github.com/dmwm/CRABServer/blob/master/src/python/ServerUtilities.py#L61-L77 + + tm_id VARCHAR(60) NOT NULL, + tm_username VARCHAR(30) NOT NULL, + tm_taskname VARCHAR(255) NOT NULL, + tm_destination VARCHAR(100) NOT NULL, + tm_destination_lfn VARCHAR(1000) NOT NULL, + tm_source VARCHAR(100) NOT NULL, + tm_source_lfn VARCHAR(1000) NOT NULL, + tm_filesize NUMBER(20) NOT NULL, + tm_publish NUMBER(1) NOT NULL, + tm_jobid NUMBER(10) NOT NULL, + tm_job_retry_count NUMBER(5), + tm_type VARCHAR(20) NOT NULL, + tm_aso_worker VARCHAR(100), + tm_transfer_retry_count NUMBER(5) DEFAULT 0, + tm_transfer_max_retry_count NUMBER(5) DEFAULT 2, + tm_publication_retry_count NUMBER(5) DEFAULT 0, + tm_publication_max_retry_count NUMBER(5) DEFAULT 2, + tm_rest_host VARCHAR(50) NOT NULL, + tm_rest_uri VARCHAR(255) NOT NULL, + tm_transfer_state NUMBER(1) NOT NULL, + tm_publication_state NUMBER(1) NOT NULL, + tm_transfer_failure_reason VARCHAR(1000), + tm_publication_failure_reason VARCHAR(1000), + tm_fts_id VARCHAR(255), + tm_fts_instance VARCHAR(255), + tm_last_update NUMBER(11) NOT NULL, + tm_start_time NUMBER(11) NOT NULL, + tm_end_time NUMBER(11) + """ + return StructType([ + StructField("tm_id", StringType(), True), + StructField("tm_username", StringType(), True), + StructField("tm_taskname", StringType(), True), + StructField("tm_destination", StringType(), True), + StructField("tm_destination_lfn", StringType(), True), + StructField("tm_source", StringType(), True), + StructField("tm_source_lfn", StringType(), True), + StructField("tm_filesize", DoubleType(), True), + StructField("tm_publish", DoubleType(), True), + StructField("tm_jobid", IntegerType(), True), + StructField("tm_job_retry_count", IntegerType(), True), + StructField("tm_type", StringType(), True), + StructField("tm_aso_worker", StringType(), True), + StructField("tm_transfer_retry_count", IntegerType(), True), + StructField("tm_transfer_max_retry_count", IntegerType(), True), + StructField("tm_publication_retry_count", IntegerType(), True), + StructField("tm_publication_max_retry_count", IntegerType(), True), + StructField("tm_rest_host", StringType(), True), + StructField("tm_rest_uri", StringType(), True), + StructField("tm_transfer_state", IntegerType(), True), + StructField("tm_publication_state", IntegerType(), True), + StructField("tm_transfer_failure_reason", StringType(), True), + StructField("tm_publication_failure_reason", StringType(), True), + StructField("tm_fts_id", StringType(), True), + StructField("tm_fts_instance", StringType(), True), + StructField("tm_last_update", DoubleType(), True), + StructField("tm_start_time", DoubleType(), True), + StructField("tm_end_time", DoubleType(), True), + ]) + + +def aggregated_data_schema(): + """ + dn: string (nullable = true) + dataset_name: string (nullable = true) + site_name: string (nullable = true) + app: string (nullable = true) + uid: integer (nullable = true) + stream: string (nullable = true) + timestamp: integer (nullable = true) + nacc: integer (nullable = true) + distinct_users: integer (nullable = true) + site_tier: string (nullable = true) + cpu_time: double (nullable = true) + wc_time: double (nullable = true) + primary_name: string (nullable = true) + processing_name: string (nullable = true) + data_tier: string (nullable = true) + """ + + return StructType([ + StructField("dn", StringType(), True), + StructField("dataset_name", StringType(), True), + StructField("site_name", StringType(), True), + StructField("app", StringType(), True), + StructField("uid", LongType(), True), + StructField("stream", StringType(), True), + StructField("timestamp", LongType(), True), + StructField("nacc", IntegerType(), True), + StructField("distinct_users", IntegerType(), True), + StructField("site_tier", StringType(), True), + StructField("cpu_time", DoubleType(), True), + StructField("wc_time", DoubleType(), True), + StructField("primary_name", StringType(), True), + StructField("processing_name", StringType(), True), + StructField("data_tier", StringType(), True), + ]) + +def schema_empty_aaa(): + """ + src_experiment_site: string (nullable = true) + user_dn: string (nullable = true) + file_lfn: string (nullable = true) + """ + return StructType([ + StructField("src_experiment_site", StringType(), True), + StructField("user_dn", StringType(), True), + StructField("file_lfn", StringType(), True), + ]) + +def schema_empty_eos(): + """ + src_experiment_site: string (nullable = true) + """ + return StructType([ + StructField("file_lfn", StringType(), True), + StructField("user_dn", StringType(), True), + StructField("host", StringType(), True), + ]) diff --git a/consistency_check/spark/setup.sh b/consistency_check/spark/setup.sh new file mode 100644 index 0000000..a939216 --- /dev/null +++ b/consistency_check/spark/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source /cvmfs/sft.cern.ch/lcg/views/LCG_95a/x86_64-centos7-gcc7-opt/setup.sh +source /cvmfs/sft.cern.ch/lcg/etc/hadoop-confext/hadoop-swan-setconf.sh analytix +kinit diff --git a/scripts/csv2xml.py b/scripts/csv2xml.py index 422b4b5..af10a1a 100644 --- a/scripts/csv2xml.py +++ b/scripts/csv2xml.py @@ -31,7 +31,7 @@ block = ET.SubElement(dataset,"block", name=key2) block.set("is-open", "n") for subitem in items2: - file = ET.SubElement(block,"file",name=subitem[2],bytes=subitem[3],checksum=subitem[4]+","+subitem[5]) + file = ET.SubElement(block,"file",name=subitem[2],bytes=subitem[3],checksum='cksum:'+subitem[4]+",adler32:"+subitem[5]) outfile = open(sys.argv[1]+'.xml', 'w') outfile.writelines(minidom.parseString(ET.tostring(root)).toprettyxml(indent = " ")) diff --git a/scripts/find_missing_datasets.py b/scripts/find_missing_datasets.py new file mode 100755 index 0000000..a168757 --- /dev/null +++ b/scripts/find_missing_datasets.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +import sys +import os +import json +import urllib2 + +## This tools uses Phedex API blockarrive +## USAGE: python find_missing_datasets.py + +def get_missing_data(): + url='https://cmsweb.cern.ch/phedex/datasvc/json/prod/nodes' + result = url_reading(url) + sites = get_node_name(result) + for site in sites: + get_block_with_no_source_replica(site) + +def get_node_name(result): + sites_list = []; + for node in result['phedex']['node']: + if "T1" in node['name'] or "T2" in node['name']: + sites_list.append(node['name']) + return sites_list + +def get_block_with_no_source_replica(site): + url='https://cmsweb.cern.ch/phedex/datasvc/json/prod/blockarrive?&basis=-6&to_node=' + site + result = url_reading(url) + with open('missing_data.txt', 'a+') as f: + f.write("%s\r\n" %site) + for block in result['phedex']['block']: + f.write("%s\r\n" %block['dataset']) + +def url_reading(url): + jstr = urllib2.urlopen(url).read() + jstr = jstr.replace("\n", " ") + result = json.loads(jstr) + return result + +def init(): + get_missing_data() + +init() diff --git a/scripts/get_routed_bytes.py b/scripts/get_routed_bytes.py index 988e930..6e02746 100644 --- a/scripts/get_routed_bytes.py +++ b/scripts/get_routed_bytes.py @@ -6,7 +6,7 @@ ## Get the volume routed to a site given the CMS site name + Phedex queue (priority) ## This tools uses Phedex API routedblocks -## USAGE: python get_routed_bytes.py +## USAGE: python get_routed_bytes.py def formatSize(size): output = '' @@ -24,20 +24,25 @@ def formatSize(size): output += "%.3f PB" % (float(size)/1E15) return output -def get_routed_bytes(site, priority): +def get_routed_bytes(site): url='https://cmsweb.cern.ch/phedex/datasvc/json/prod/routedblocks?to=' + site jstr = urllib2.urlopen(url).read() jstr = jstr.replace("\n", " ") result = json.loads(jstr) + all_priorities = ["low", "normal", "high", "reserved"] bytes = 0 + for priority in all_priorities: + get_priority(result,priority,bytes) +def get_priority(result,priority,bytes): for route in result['phedex']['route']: if (route['priority'] == priority): for block in route['block']: bytes += float(block['route_bytes']) - print formatSize(bytes) + final_record = priority + " "+ formatSize(bytes) + print final_record def init(): - get_routed_bytes(str(sys.argv[1]), str(sys.argv[2])) + get_routed_bytes(str(sys.argv[1])) init() diff --git a/scripts/setup.sh b/scripts/setup.sh index adf0fcd..e3e3cc8 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -2,6 +2,6 @@ cwd=$PWD source ~/phedex/PHEDEX/etc/profile.d/env.sh cd ~/work/public/CMSSW_7_4_15/src cmsenv -source /cvmfs/cms.cern.ch/crab/crab.sh +source /cvmfs/cms.cern.ch/crab3/crab.sh voms-proxy-init -voms cms cd $cwd diff --git a/storage_overview/overview.py b/storage_overview/overview.py index fdc4d6c..388c5cc 100755 --- a/storage_overview/overview.py +++ b/storage_overview/overview.py @@ -54,22 +54,22 @@ ) # TB PLEDGES = { - "T0_CH_CERN_MSS": 70500, #2016 pledge 44000 - "T1_DE_KIT_MSS": 19500, #2016 pledge 10000 - "T1_ES_PIC_MSS": 6885, #2016 pledge 5100 - "T1_FR_CCIN2P3_MSS": 11000, #2016 pledge 8100 - "T1_IT_CNAF_MSS": 21000, #early 2017 16000 #2016 pledge 12000 - "T1_RU_JINR_MSS": 8000, #2016 pledge 5000 - "T1_UK_RAL_MSS": 12720, #2016 pledge 8000 - "T1_US_FNAL_MSS": 54000, #2016 pledge 40000 - "T1_DE_KIT_Disk": 5140, #until early 2017 3500 - "T1_ES_PIC_Disk": 2499, #until early 2017 1683 - "T1_FR_CCIN2P3_Disk": 3600, #until early 2017 2700 - "T1_IT_CNAF_Disk": 5880, #until early 2017 3960 - "T1_RU_JINR_Disk": 3900, #early 2017 4000 #2800,# SE(2800TB) + cache for tapes(400TB) - "T1_UK_RAL_Disk": 4304, #early 2027 2640 - "T1_US_FNAL_Disk": 19600, # pledge 13200 until April 1 2017 - "T2_CH_CERN": 24600 # early 2017 5370 + "T0_CH_CERN_MSS": 97000, #2017 pledge70500, #2016 pledge 44000 + "T1_DE_KIT_MSS": 18800, #2017 pledge 19500, #2016 pledge 10000 + "T1_ES_PIC_MSS": 8629, #2017 pledge 6885, #2016 pledge 5100 + "T1_FR_CCIN2P3_MSS": 14500,#2017 pledge 11000, #2016 pledge 8100 + "T1_IT_CNAF_MSS": 24440, #2017 pledge 21000, #early 2017 16000 #2016 pledge 12000 + "T1_RU_JINR_MSS": 9000, #2017 pledge 8000, #2016 pledge 5000 + "T1_UK_RAL_MSS": 14936, #2017 pledge 12720, #2016 pledge 8000 + "T1_US_FNAL_MSS": 75200, #2017 pledge 54000, #2016 pledge 40000 + "T1_DE_KIT_Disk": 6000, #2017 pledge 5140, #until early 2017 3500 + "T1_ES_PIC_Disk": 2754, #2017 pledge 2499, #until early 2017 1683 + "T1_FR_CCIN2P3_Disk": 4600, #2017 pledge 3600, #until early 2017 2700 + "T1_IT_CNAF_Disk": 7200, #2017 pledge 5880, #until early 2017 3960 + "T1_RU_JINR_Disk": 6000, #2017 pledge 3900, #early 2017 4000 #2800,# SE(2800TB) + cache for tapes(400TB) + "T1_UK_RAL_Disk": 4784, #2017 pledge 4304, #early 2027 2640 + "T1_US_FNAL_Disk": 24000, #2017 pledge 19600, # pledge 13200 until April 1 2017 + "T2_CH_CERN": 26100, #2017 pledge 24600 # early 2017 5370 } T0CERNPLEDGE=44000 T0CERNUSED=31524.8*UNIT @@ -400,7 +400,10 @@ def meetingSummary(self, isTape=False): #row.append(self._create_cell(to_TB(PLEDGES[FNALTAPE]*UNIT - T1FNALMSSUSED))) #grand_total += T1FNALMSSUSED ''' - if site.name == "T0_CH_CERN_MSS" or site.name == "T1_DE_KIT_MSS" or site.name == "T1_DE_KIT_Disk" or site.name == "T1_IT_CNAF_MSS" or site.name == "T1_IT_CNAF_Disk": + #sites_set_unusable = ["T0_CH_CERN_MSS", "T1_IT_CNAF_Disk", "T1_IT_CNAF_MSS", "T1_DE_KIT_Disk", "T1_DE_KIT_MSS"] + sites_set_unusable = [] + + if site.name in sites_set_unusable: row.append(self._create_cell(to_TB(0))) else: row.append(self._create_cell(to_TB(total))) @@ -444,13 +447,12 @@ def jsonSummary(self, isTape=False): jsonsummary['Free'][site.name] = int((PLEDGES[site.name]*UNIT - T1FNALMSSUSED)/UNIT) for site in jsonsummary['Free']: - ''' - if (site == 'T1_US_FNAL_MSS' or site =='T1_DE_KIT_MSS'): - jsonsummary['Usable'][site] = 0 - ''' - if (site == 'T0_CH_CERN_MSS'): + #sites_set_unusable = ["T0_CH_CERN_MSS", "T1_IT_CNAF_Disk", "T1_IT_CNAF_MSS", "T1_DE_KIT_Disk", "T1_DE_KIT_MSS"] + sites_set_unusable = ["T0_CH_CERN_MSS"] + + if site in sites_set_unusable: jsonsummary['Usable'][site] = 0 - else: + else: jsonsummary['Usable'][site]=jsonsummary['Free'][site] #uncoment this part when there is no blacklisting to any site diff --git a/test b/test new file mode 100644 index 0000000..e69de29