XRootD
Loading...
Searching...
No Matches
XrdClUtils.cc
Go to the documentation of this file.
1//------------------------------------------------------------------------------
2// Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN)
3// Author: Lukasz Janyst <ljanyst@cern.ch>
4//------------------------------------------------------------------------------
5// XRootD is free software: you can redistribute it and/or modify
6// it under the terms of the GNU Lesser General Public License as published by
7// the Free Software Foundation, either version 3 of the License, or
8// (at your option) any later version.
9//
10// XRootD is distributed in the hope that it will be useful,
11// but WITHOUT ANY WARRANTY; without even the implied warranty of
12// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13// GNU General Public License for more details.
14//
15// You should have received a copy of the GNU Lesser General Public License
16// along with XRootD. If not, see <http://www.gnu.org/licenses/>.
17//------------------------------------------------------------------------------
18
19#include "XrdCl/XrdClUtils.hh"
25#include "XrdCl/XrdClMessage.hh"
27#include "XrdNet/XrdNetAddr.hh"
28
29#include <algorithm>
30#include <iomanip>
31#include <sstream>
32#include <fstream>
33#include <functional>
34#include <cctype>
35#include <locale>
36#include <map>
37#include <string>
38#include <set>
39#include <cctype>
40#include <random>
41#include <chrono>
42
43#include <sys/types.h>
44#include <dirent.h>
45
46#if __cplusplus < 201103L
47#include <ctime>
48#endif
49
50namespace
51{
52 bool isNotSpace( char c )
53 {
54 return c != ' ';
55 }
56
57 //----------------------------------------------------------------------------
58 // Ordering function for sorting IP addresses
59 //----------------------------------------------------------------------------
60 struct PreferIPv6
61 {
62 bool operator() ( const XrdNetAddr &l, const XrdNetAddr &r )
63 {
64 bool rIsIPv4 = false;
67 rIsIPv4 = true;
68
69 if( l.isIPType( XrdNetAddrInfo::IPv6 ) && rIsIPv4 )
70 return true;
71 return false;
72 }
73 };
74}
75
76namespace XrdCl
77{
78 //----------------------------------------------------------------------------
79 // Get a parameter either from the environment or URL
80 //----------------------------------------------------------------------------
81 int Utils::GetIntParameter( const URL &url,
82 const std::string &name,
83 int defaultVal )
84 {
85 Env *env = DefaultEnv::GetEnv();
86 int value = defaultVal;
87 char *endPtr;
88 URL::ParamsMap::const_iterator it;
89
90 env->GetInt( name, value );
91 it = url.GetParams().find( std::string("XrdCl.") + name );
92 if( it != url.GetParams().end() )
93 {
94 int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 );
95 if( !*endPtr )
96 value = urlValue;
97 }
98 return value;
99 }
100
101 //----------------------------------------------------------------------------
102 // Get a parameter either from the environment or URL
103 //----------------------------------------------------------------------------
104 std::string Utils::GetStringParameter( const URL &url,
105 const std::string &name,
106 const std::string &defaultVal )
107 {
108 Env *env = DefaultEnv::GetEnv();
109 std::string value = defaultVal;
110 URL::ParamsMap::const_iterator it;
111
112 env->GetString( name, value );
113 it = url.GetParams().find( std::string("XrdCl.") + name );
114 if( it != url.GetParams().end() )
115 value = it->second;
116
117 return value;
118 }
119
120 //----------------------------------------------------------------------------
121 // Interpret a string as address type, default to IPAll
122 //----------------------------------------------------------------------------
123 Utils::AddressType Utils::String2AddressType( const std::string &addressType )
124 {
125 if( addressType == "IPv6" )
126 return IPv6;
127 else if( addressType == "IPv4" )
128 return IPv4;
129 else if( addressType == "IPv4Mapped6" )
130 return IPv4Mapped6;
131 else if( addressType == "IPAll" )
132 return IPAll;
133 else
134 return IPAuto;
135 }
136
137 //----------------------------------------------------------------------------
138 // Resolve IP addresses
139 //----------------------------------------------------------------------------
140 Status Utils::GetHostAddresses( std::vector<XrdNetAddr> &addresses,
141 const URL &url,
142 Utils::AddressType type )
143 {
144 Log *log = DefaultEnv::GetLog();
145 const char *err = 0;
146 int ordn;
147
148 //--------------------------------------------------------------------------
149 // Resolve all the addresses
150 //--------------------------------------------------------------------------
151 std::ostringstream o; o << url.GetHostName() << ":" << url.GetPort();
153
154 if( type == IPv6 ) opts = XrdNetUtils::onlyIPv6;
155 else if( type == IPv4 ) opts = XrdNetUtils::onlyIPv4;
156 else if( type == IPv4Mapped6 ) opts = XrdNetUtils::allV4Map;
157 else if( type == IPAll ) opts = XrdNetUtils::allIPMap;
159
160 //--------------------------------------------------------------------------
161 // Check what are the preferences IPv6 or IPv4
162 //--------------------------------------------------------------------------
163 int preferIPv4 = DefaultPreferIPv4;
164 DefaultEnv::GetEnv()->GetInt( "PreferIPv4", preferIPv4 );
165
166 //--------------------------------------------------------------------------
167 // Partition the addresses according to the preferences
168 //
169 // The preferred IP family goes to the back as it is easier to remove
170 // items from the back of the vector
171 //--------------------------------------------------------------------------
173
174 //--------------------------------------------------------------------------
175 // Now get all of the properly partitioned addresses; ordn will hold the
176 // number of non-preferred addresses at the front of the table.
177 //--------------------------------------------------------------------------
178 err = XrdNetUtils::GetAddrs( o.str(), addresses, &ordn, opts );
179
180 if( err )
181 {
182 log->Error( UtilityMsg, "Unable to resolve %s: %s", o.str().c_str(),
183 err );
184 return Status( stError, errInvalidAddr );
185 }
186
187 if( addresses.size() == 0 )
188 {
189 log->Error( UtilityMsg, "No addresses for %s were found",
190 o.str().c_str() );
191 return Status( stError, errInvalidAddr );
192 }
193
194 //--------------------------------------------------------------------------
195 // Shuffle each partition
196 //--------------------------------------------------------------------------
197
198 int ipNoShuffle = DefaultIPNoShuffle;
199 Env *env = DefaultEnv::GetEnv();
200 env->GetInt( "IPNoShuffle", ipNoShuffle );
201
202 if( !ipNoShuffle )
203 {
204#if __cplusplus < 201103L
205 // initialize the random generator only once
206 static struct only_once_t
207 {
208 only_once_t()
209 {
210 std::srand ( unsigned ( std::time(0) ) );
211 }
212 } only_once;
213
214 std::random_shuffle( addresses.begin(), addresses.begin() + ordn );
215 std::random_shuffle( addresses.begin() + ordn, addresses.end() );
216#else
217 static std::default_random_engine rand_engine(
218 std::chrono::system_clock::now().time_since_epoch().count() );
219
220 std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine );
221 std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine );
222#endif
223 }
224
225 //--------------------------------------------------------------------------
226 // Return status as the result is already in the output parameter
227 //--------------------------------------------------------------------------
228 return Status();
229 }
230
231 //----------------------------------------------------------------------------
232 // Log all the addresses on the list
233 //----------------------------------------------------------------------------
235 uint64_t type,
236 const std::string &hostId,
237 std::vector<XrdNetAddr> &addresses )
238 {
239 std::string addrStr;
240 std::vector<XrdNetAddr>::iterator it;
241 for( it = addresses.begin(); it != addresses.end(); ++it )
242 {
243 char nameBuff[256];
244 it->Format( nameBuff, 256, XrdNetAddrInfo::fmtAdv6 );
245 addrStr += nameBuff;
246 addrStr += ", ";
247 }
248 addrStr.erase( addrStr.length()-2, 2 );
249 log->Debug( type, "[%s] Found %zu address(es): %s",
250 hostId.c_str(), addresses.size(), addrStr.c_str() );
251 }
252
253 //----------------------------------------------------------------------------
254 // Convert timestamp to a string
255 //----------------------------------------------------------------------------
256 std::string Utils::TimeToString( time_t timestamp )
257 {
258 char now[30];
259 tm tsNow;
260 time_t ttNow = timestamp;
261 localtime_r( &ttNow, &tsNow );
262 strftime( now, 30, "%Y-%m-%d %H:%M:%S %z", &tsNow );
263 return now;
264 }
265
266 //----------------------------------------------------------------------------
267 // Get the elapsed microseconds between two timevals
268 //----------------------------------------------------------------------------
269 uint64_t Utils::GetElapsedMicroSecs( timeval start, timeval end )
270 {
271 uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec;
272 uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec;
273 return endUSec-startUSec;
274 }
275
276 //----------------------------------------------------------------------------
277 // Get remote checksum
278 //----------------------------------------------------------------------------
280 const std::string &checkSumType,
281 const URL &url )
282 {
283 FileSystem *fs = new FileSystem( url );
284 // add the 'cks.type' cgi tag in order to
285 // select the proper checksum type in case
286 // the server supports more than one checksum
287 size_t pos = url.GetPath().find( '?' );
288 std::string cksPath = url.GetPath() + ( pos == std::string::npos ? '?' : '&' ) + "cks.type=" + checkSumType;
289 Buffer arg; arg.FromString( cksPath );
290 Buffer *cksResponse = 0;
291 XRootDStatus st;
292 Log *log = DefaultEnv::GetLog();
293
294 st = fs->Query( QueryCode::Checksum, arg, cksResponse );
295 delete fs;
296
297 if( !st.IsOK() )
298 {
299 std::string msg = st.GetErrorMessage();
300 msg += " Got an error while querying the checksum!";
301 st.SetErrorMessage( msg );
302 return st;
303 }
304
305 if( !cksResponse )
306 return XRootDStatus( stError, errInternal, 0, "Got invalid response while querying the checksum!" );
307
308 std::vector<std::string> elems;
309 Utils::splitString( elems, cksResponse->ToString(), " " );
310 delete cksResponse;
311
312 if( elems.size() != 2 )
313 return XRootDStatus( stError, errInvalidResponse, 0, "Got invalid response while querying the checksum!" );
314
315 if( elems[0] != checkSumType )
317
318 checkSum = elems[0] + ":";
319 checkSum += NormalizeChecksum( elems[0], elems[1] );
320
321 log->Dump( UtilityMsg, "Checksum for %s checksum: %s",
322 url.GetPath().c_str(), checkSum.c_str() );
323
324 return XRootDStatus();
325 }
326
327 //------------------------------------------------------------------------
328 // Get a checksum from local file
329 //------------------------------------------------------------------------
330 XRootDStatus Utils::GetLocalCheckSum( std::string &checkSum,
331 const std::string &checkSumType,
332 const std::string &path )
333 {
334 Log *log = DefaultEnv::GetLog();
336
337 if( !cksMan )
338 {
339 log->Error( UtilityMsg, "Unable to get the checksum manager" );
341 }
342
343 XrdCksData ckSum; ckSum.Set( checkSumType.c_str() );
344 bool status = cksMan->Calculate( ckSum, checkSumType, path.c_str() );
345 if( !status )
346 {
347 log->Error( UtilityMsg, "Error while calculating checksum for %s",
348 path.c_str() );
350 }
351
352 char *cksBuffer = new char[265];
353 ckSum.Get( cksBuffer, 256 );
354 checkSum = checkSumType + ":";
355 checkSum += NormalizeChecksum( checkSumType, cksBuffer );
356 delete [] cksBuffer;
357
358 log->Dump( UtilityMsg, "Checksum for %s is: %s", path.c_str(),
359 checkSum.c_str() );
360
361 return XRootDStatus();
362 }
363
364 //----------------------------------------------------------------------------
365 // Convert bytes to a human readable string
366 //----------------------------------------------------------------------------
367 std::string Utils::BytesToString( uint64_t bytes )
368 {
369 double final = bytes;
370 int i = 0;
371 char suf[3] = { 'k', 'M', 'G' };
372 for( i = 0; i < 3 && final > 1024; ++i, final /= 1024 ) {};
373 std::ostringstream o;
374 o << std::setprecision(4) << final;
375 if( i > 0 ) o << suf[i-1];
376 return o.str();
377 }
378
379 //----------------------------------------------------------------------------
380 // Check if peer supports tpc
381 //----------------------------------------------------------------------------
382 XRootDStatus Utils::CheckTPC( const std::string &server, uint16_t timeout )
383 {
384 Log *log = DefaultEnv::GetLog();
385 log->Debug( UtilityMsg, "Checking if the data server %s supports tpc",
386 server.c_str() );
387
388 FileSystem sourceDSFS( server );
389 Buffer queryArg; queryArg.FromString( "tpc" );
390 Buffer *queryResponse = 0;
391 XRootDStatus st;
392 st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
393 timeout );
394 if( !st.IsOK() )
395 {
396 log->Error( UtilityMsg, "Cannot query source data server %s: %s",
397 server.c_str(), st.ToStr().c_str() );
398 st.status = stFatal;
399 return st;
400 }
401
402 if( !queryResponse )
403 {
404 log->Error( UtilityMsg, "Cannot query source data server: empty response." );
405 st.status = stFatal;
406 return st;
407 }
408
409 std::string answer = queryResponse->ToString();
410 delete queryResponse;
411 if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0)
412 {
413 log->Debug( UtilityMsg, "Third party copy not supported at: %s",
414 server.c_str() );
416 }
417 log->Debug( UtilityMsg, "Third party copy supported at: %s",
418 server.c_str() );
419
420 return XRootDStatus();
421 }
422
423 //------------------------------------------------------------------------
424 // Check if peer supports tpc / tpc lite
425 //------------------------------------------------------------------------
426 XRootDStatus Utils::CheckTPCLite( const std::string &server, uint16_t timeout )
427 {
428 Log *log = DefaultEnv::GetLog();
429 log->Debug( UtilityMsg, "Checking if the data server %s supports tpc / tpc lite",
430 server.c_str() );
431
432 FileSystem sourceDSFS( server );
433 Buffer queryArg; queryArg.FromString( "tpc tpcdlg" );
434 Buffer *queryResponse = 0;
435 XRootDStatus st;
436 st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
437 timeout );
438 if( !st.IsOK() )
439 {
440 log->Error( UtilityMsg, "Cannot query source data server %s: %s",
441 server.c_str(), st.ToStr().c_str() );
442 st.status = stFatal;
443 return st;
444 }
445
446 if( !queryResponse )
447 {
448 log->Error( UtilityMsg, "Cannot query source data server: empty response." );
449 st.status = stFatal;
450 return st;
451 }
452
453 std::string answer = queryResponse->ToString();
454 delete queryResponse;
455
456 if( answer.empty() )
457 {
458 log->Error( UtilityMsg, "Cannot query source data server: empty response." );
459 st.status = stFatal;
460 return st;
461 }
462
463 std::vector<std::string> resp;
464 Utils::splitString( resp, answer, "\n" );
465
466 if( resp.empty() || resp[0].empty() ||
467 !isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 )
468 {
469 log->Debug( UtilityMsg, "Third party copy not supported at: %s",
470 server.c_str() );
472 }
473
474 if( resp.size() == 1 || resp[1] == "tpcdlg" )
475 {
476 log->Debug( UtilityMsg, "TPC lite not supported at: %s",
477 server.c_str() );
478 return XRootDStatus( stOK, suPartial );
479 }
480
481 log->Debug( UtilityMsg, "TPC lite supported at: %s",
482 server.c_str() );
483
484 return XRootDStatus();
485 }
486
487 //----------------------------------------------------------------------------
488 // Convert the fully qualified host name to country code
489 //----------------------------------------------------------------------------
490 std::string Utils::FQDNToCC( const std::string &fqdn )
491 {
492 std::vector<std::string> el;
493 Utils::splitString( el, fqdn, "." );
494 if( el.size() < 2 )
495 return "us";
496
497 std::string cc = *el.rbegin();
498 if( cc.length() == 2 )
499 return cc;
500 return "us";
501 }
502
503 //----------------------------------------------------------------------------
504 // Get directory entries
505 //----------------------------------------------------------------------------
506 Status Utils::GetDirectoryEntries( std::vector<std::string> &entries,
507 const std::string &path )
508 {
509 DIR *dp = opendir( path.c_str() );
510 if( !dp )
511 return Status( stError, errOSError, errno );
512
513 dirent *dirEntry;
514
515 while( (dirEntry = readdir(dp)) != 0 )
516 {
517 std::string entryName = dirEntry->d_name;
518 if( !entryName.compare( 0, 2, "..") )
519 continue;
520 if( !entryName.compare( 0, 1, ".") )
521 continue;
522
523 entries.push_back( dirEntry->d_name );
524 }
525
526 closedir(dp);
527
528 return Status();
529 }
530
531 //----------------------------------------------------------------------------
532 // Process a config file and return key-value pairs
533 //----------------------------------------------------------------------------
534 Status Utils::ProcessConfig( std::map<std::string, std::string> &config,
535 const std::string &file )
536 {
537 config.clear();
538 std::ifstream inFile( file.c_str() );
539 if( !inFile.good() )
540 return Status( stError, errOSError, errno );
541
542 errno = 0;
543 std::string line;
544 while( std::getline( inFile, line ) )
545 {
546 if( line.empty() || line[0] == '#' )
547 continue;
548
549 std::vector<std::string> elems;
550 splitString( elems, line, "=" );
551 if( elems.size() != 2 )
552 return Status( stError, errConfig );
553 std::string key = elems[0]; Trim( key );
554 std::string value = elems[1]; Trim( value );
555 config[key] = value;
556 }
557
558 if( errno )
559 return Status( stError, errOSError, errno );
560 return Status();
561 }
562
563 //------------------------------------------------------------------------
565 //------------------------------------------------------------------------
566 Status Utils::ProcessConfigDir( std::map<std::string, std::string> &config,
567 const std::string &dir )
568 {
569 Log *log = DefaultEnv::GetLog();
570 log->Debug( UtilityMsg, "Processing configuration files in %s...",
571 dir.c_str());
572
573 std::vector<std::string> entries;
574 Status st = Utils::GetDirectoryEntries( entries, dir );
575 if( !st.IsOK() )
576 {
577 log->Debug( UtilityMsg, "Unable to process directory %s: %s",
578 dir.c_str(), st.ToString().c_str() );
579 return st;
580 }
581
582 static const std::string suffix = ".conf";
583 for( auto &entry : entries )
584 {
585 std::string confFile = dir + "/" + entry;
586
587 if( confFile.length() <= suffix.length() )
588 continue;
589 if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) )
590 continue;
591
592 st = ProcessConfig( config, confFile );
593 if( !st.IsOK() )
594 {
595 log->Debug( UtilityMsg, "Unable to process configuration file %s: %s",
596 confFile.c_str(), st.ToString().c_str() );
597 }
598 }
599
600 return Status();
601 }
602
603 //----------------------------------------------------------------------------
604 // Trim a string
605 //----------------------------------------------------------------------------
606 void Utils::Trim( std::string &str )
607 {
608 str.erase( str.begin(),
609 std::find_if( str.begin(), str.end(), isNotSpace ) );
610 str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(),
611 str.end() );
612 }
613
614 //----------------------------------------------------------------------------
615 // Log property list
616 //----------------------------------------------------------------------------
618 uint64_t topic,
619 const char *format,
620 const PropertyList &list )
621 {
622 if( unlikely(log->GetLevel() >= Log::DumpMsg) ) {
623 PropertyList::PropertyMap::const_iterator it;
624 std::string keyVals;
625 for (it = list.begin(); it != list.end(); ++it)
626 keyVals += "'" + it->first + "' = '" + obfuscateAuth(it->second) + "', ";
627 keyVals.erase(keyVals.length() - 2, 2);
628 log->Dump(topic, format, keyVals.c_str());
629 }
630 }
631
632 //----------------------------------------------------------------------------
633 // Print a char array as hex
634 //----------------------------------------------------------------------------
635 std::string Utils::Char2Hex( uint8_t *array, uint16_t size )
636 {
637 char *hex = new char[2*size+1];
638 for( uint16_t i = 0; i < size; ++i )
639 snprintf( hex+(2*i), 3, "%02x", (int)array[i] );
640 std::string result = hex;
641 delete [] hex;
642 return result;
643 }
644
645 //----------------------------------------------------------------------------
646 // Normalize checksum
647 //----------------------------------------------------------------------------
648 std::string Utils::NormalizeChecksum( const std::string &name,
649 const std::string &checksum )
650 {
651 if( name == "adler32" || name == "crc32" )
652 {
653 size_t i;
654 for( i = 0; i < checksum.length(); ++i )
655 if( checksum[i] != '0' )
656 break;
657 return checksum.substr(i);
658 }
659 return checksum;
660 }
661
662 //----------------------------------------------------------------------------
663 // Get supported checksum types for given URL
664 //----------------------------------------------------------------------------
665 std::vector<std::string> Utils::GetSupportedCheckSums( const XrdCl::URL &url )
666 {
667 std::vector<std::string> ret;
668
669 FileSystem fs( url );
670 Buffer arg; arg.FromString( "chksum" );
671 Buffer *resp = 0;
672 XRootDStatus st = fs.Query( QueryCode::Config, arg, resp );
673 if( st.IsOK() )
674 {
675 std::string response = resp->ToString();
676 if( response != "chksum" )
677 {
678 // we are expecting a response of format: '0:zcrc32,1:adler32'
679 std::vector<std::string> result;
680 Utils::splitString( result, response, "," );
681
682 std::vector<std::string>::iterator itr = result.begin();
683 for( ; itr != result.end(); ++itr )
684 {
685 size_t pos = itr->find( ':' );
686 if( pos == std::string::npos ) continue;
687 std::string cksname = itr->substr( pos + 1 );
688 // remove all white spaces
689 cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ),
690 cksname.end() );
691 ret.push_back( std::move( cksname ) );
692 }
693 }
694 }
695
696 return ret;
697 }
698
699
700 //------------------------------------------------------------------------
702 //------------------------------------------------------------------------
703 bool Utils::CheckEC( const Message *req, const URL &url )
704 {
705#ifdef WITH_XRDEC
706 // make sure that if we will be writing it is a new file
707 ClientRequest *request = (ClientRequest*)req->GetBuffer();
708 uint16_t options = ntohs( request->open.options );
709 bool open_wrt = ( options & kXR_open_updt ) || ( options & kXR_open_wrto );
710 bool open_new = ( options & kXR_new );
711 if( open_wrt && !open_new ) return false;
712
713 const URL::ParamsMap &params = url.GetParams();
714 // make sure all the xrdec. tokens are present and the values are sane
715 URL::ParamsMap::const_iterator itr = params.find( "xrdec.nbdta" );
716 if( itr == params.end() ) return false;
717 size_t nbdta = std::stoul( itr->second );
718
719 itr = params.find( "xrdec.nbprt" );
720 if( itr == params.end() ) return false;
721 size_t nbprt = std::stoul( itr->second );
722
723 itr = params.find( "xrdec.blksz" );
724 if( itr == params.end() ) return false;
725
726 itr = params.find( "xrdec.plgr" );
727 if( itr == params.end() ) return false;
728 std::vector<std::string> plgr;
729 splitString( plgr, itr->second, "," );
730 if( plgr.size() < nbdta + nbprt ) return false;
731
732 itr = params.find( "xrdec.objid" );
733 if( itr == params.end() ) return false;
734
735 itr = params.find( "xrdec.format" );
736 if( itr == params.end() ) return false;
737 size_t format = std::stoul( itr->second );
738 if( format != 1 ) return false; // TODO use constant
739
740 itr = params.find( "xrdec.dtacgi" );
741 if( itr != params.end() )
742 {
743 std::vector<std::string> dtacgi;
744 splitString( dtacgi, itr->second, "," );
745 if( plgr.size() != dtacgi.size() ) return false;
746 }
747
748 itr = params.find( "xrdec.mdtacgi" );
749 if( itr != params.end() )
750 {
751 std::vector<std::string> mdtacgi;
752 splitString( mdtacgi, itr->second, "," );
753 if( plgr.size() != mdtacgi.size() ) return false;
754 }
755
756 itr = params.find( "xrdec.cosc" );
757 if( itr == params.end() ) return false;
758 std::string cosc = itr->second;
759 if( cosc != "true" && cosc != "false" ) return false;
760
761 return true;
762#else
763 return false;
764#endif
765 }
766
767
768 //----------------------------------------------------------------------------
770 //----------------------------------------------------------------------------
771 std::string Utils::InferChecksumType( const XrdCl::URL &source,
772 const XrdCl::URL &destination,
773 bool zip)
774 {
775 //--------------------------------------------------------------------------
776 // If both files are local we won't be checksumming at all
777 //--------------------------------------------------------------------------
778 if( source.IsLocalFile() && !source.IsMetalink() && destination.IsLocalFile() ) return std::string();
779
780 // checksums supported by local files
781 std::set<std::string> local_supported;
782 local_supported.insert( "adler32" );
783 local_supported.insert( "crc32" );
784 local_supported.insert( "md5" );
785 local_supported.insert( "zcrc32" );
786
787 std::vector<std::string> srccks;
788
789 if( source.IsMetalink() )
790 {
791 int useMtlnCksum = DefaultZipMtlnCksum;
792 Env *env = DefaultEnv::GetEnv();
793 env->GetInt( "ZipMtlnCksum", useMtlnCksum );
794
795 //------------------------------------------------------------------------
796 // In case of ZIP use other checksums than zcrc32 only if the user
797 // requested it explicitly.
798 //------------------------------------------------------------------------
799 if( !zip || ( zip && useMtlnCksum ) )
800 {
802 VirtualRedirector *redirector = registry.Get( source );
803 std::vector<std::string> cks = redirector->GetSupportedCheckSums();
804 srccks.insert( srccks.end(), cks.begin(), cks.end() );
805 }
806 }
807
808 if( zip )
809 {
810 //------------------------------------------------------------------------
811 // In case of ZIP we can always extract the checksum from the archive
812 //------------------------------------------------------------------------
813 srccks.push_back( "zcrc32" );
814 }
815 else if( source.GetProtocol() == "root" || source.GetProtocol() == "xroot" )
816 {
817 //------------------------------------------------------------------------
818 // If the source is a remote endpoint query the supported checksums
819 //------------------------------------------------------------------------
820 std::vector<std::string> cks = GetSupportedCheckSums( source );
821 srccks.insert( srccks.end(), cks.begin(), cks.end() );
822 }
823
824 std::vector<std::string> dstcks;
825
826 if( destination.GetProtocol() == "root" ||
827 destination.GetProtocol() == "xroot" )
828 {
829 //------------------------------------------------------------------------
830 // If the destination is a remote endpoint query the supported checksums
831 //------------------------------------------------------------------------
832 std::vector<std::string> cks = GetSupportedCheckSums( destination );
833 dstcks.insert( dstcks.end(), cks.begin(), cks.end() );
834 }
835
836 //--------------------------------------------------------------------------
837 // Now we have all the information we need, we can infer the right checksum
838 // type!!!
839 //
840 // First check if source is local
841 //--------------------------------------------------------------------------
842 if( source.IsLocalFile() && !source.IsMetalink() )
843 {
844 std::vector<std::string>::iterator itr = dstcks.begin();
845 for( ; itr != dstcks.end(); ++itr )
846 if( local_supported.count( *itr ) ) return *itr;
847 return std::string();
848 }
849
850 //--------------------------------------------------------------------------
851 // then check if destination is local
852 //--------------------------------------------------------------------------
853 if( destination.IsLocalFile() )
854 {
855 std::vector<std::string>::iterator itr = srccks.begin();
856 for( ; itr != srccks.end(); ++itr )
857 if( local_supported.count( *itr ) ) return *itr;
858 return std::string();
859 }
860
861 //--------------------------------------------------------------------------
862 // if both source and destination are remote look for a checksum that can
863 // satisfy both
864 //--------------------------------------------------------------------------
865 std::set<std::string> dst_supported( dstcks.begin(), dstcks.end() );
866 std::vector<std::string>::iterator itr = srccks.begin();
867 for( ; itr != srccks.end(); ++itr )
868 if( dst_supported.count( *itr ) ) return *itr;
869 return std::string();
870 }
871
872 //----------------------------------------------------------------------------
874 //----------------------------------------------------------------------------
875 void Utils::SplitChunks( std::vector<ChunkList> &listsvec,
876 const ChunkList &chunks,
877 const uint32_t maxcs,
878 const size_t maxc )
879 {
880 listsvec.clear();
881 if( !chunks.size() ) return;
882
883 listsvec.emplace_back();
884 ChunkList *c = &listsvec.back();
885 const size_t cs = chunks.size();
886 size_t idx = 0;
887 size_t nc = 0;
888 ChunkInfo tmpc;
889
890 c->reserve( cs );
891
892 while( idx < cs )
893 {
894 if( maxc && nc >= maxc )
895 {
896 listsvec.emplace_back();
897 c = &listsvec.back();
898 c->reserve( cs - idx );
899 nc = 0;
900 }
901
902 if( tmpc.length == 0 )
903 tmpc = chunks[idx];
904
905 if( maxcs && tmpc.length > maxcs )
906 {
907 c->emplace_back( tmpc.offset, maxcs, tmpc.buffer );
908 tmpc.offset += maxcs;
909 tmpc.length -= maxcs;
910 tmpc.buffer = static_cast<char*>( tmpc.buffer ) + maxcs;
911 }
912 else
913 {
914 c->emplace_back( tmpc.offset, tmpc.length, tmpc.buffer );
915 tmpc.length = 0;
916 ++idx;
917 }
918 ++nc;
919 }
920 }
921}
kXR_unt16 options
Definition XProtocol.hh:481
@ kXR_open_wrto
Definition XProtocol.hh:469
@ kXR_open_updt
Definition XProtocol.hh:457
@ kXR_new
Definition XProtocol.hh:455
struct ClientOpenRequest open
Definition XProtocol.hh:860
#define unlikely(x)
std::string obfuscateAuth(const std::string &input)
#define opendir(a)
Definition XrdPosix.hh:78
#define closedir(a)
Definition XrdPosix.hh:50
#define readdir(a)
Definition XrdPosix.hh:86
struct myOpts opts
int Set(const char *csName)
Definition XrdCksData.hh:81
int Get(char *Buff, int Blen)
Definition XrdCksData.hh:69
Binary blob representation.
void FromString(const std::string str)
Fill the buffer from a string.
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
std::string ToString() const
Convert the buffer to a string.
Manage the checksum calc objects.
bool Calculate(XrdCksData &result, const std::string &algName, const std::string &filePath)
Calculate a checksum of for a given file.
static CheckSumManager * GetCheckSumManager()
Get checksum manager.
static Log * GetLog()
Get default log.
static Env * GetEnv()
Get default client environment.
bool GetString(const std::string &key, std::string &value)
Definition XrdClEnv.cc:31
bool GetInt(const std::string &key, int &value)
Definition XrdClEnv.cc:89
Send file/filesystem queries to an XRootD cluster.
XRootDStatus Query(QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
Handle diagnostics.
Definition XrdClLog.hh:101
@ DumpMsg
print details of the request and responses
Definition XrdClLog.hh:113
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition XrdClLog.cc:231
LogLevel GetLevel() const
Get the log level.
Definition XrdClLog.hh:258
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
Definition XrdClLog.cc:299
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition XrdClLog.cc:282
The message representation used throughout the system.
A key-value pair map storing both keys and values as strings.
PropertyMap::const_iterator end() const
Get the end iterator.
PropertyMap::const_iterator begin() const
Get the begin iterator.
Singleton access to URL to virtual redirector mapping.
static RedirectorRegistry & Instance()
Returns reference to the single instance.
VirtualRedirector * Get(const URL &url) const
Get a virtual redirector associated with the given URL.
URL representation.
Definition XrdClURL.hh:31
const std::string & GetPath() const
Get the path.
Definition XrdClURL.hh:217
bool IsMetalink() const
Is it a URL to a metalink.
Definition XrdClURL.cc:465
std::map< std::string, std::string > ParamsMap
Definition XrdClURL.hh:33
const std::string & GetHostName() const
Get the name of the target host.
Definition XrdClURL.hh:170
bool IsLocalFile() const
Definition XrdClURL.cc:474
const ParamsMap & GetParams() const
Get the URL params.
Definition XrdClURL.hh:244
const std::string & GetProtocol() const
Get the protocol.
Definition XrdClURL.hh:118
int GetPort() const
Get the target port.
Definition XrdClURL.hh:188
static std::string TimeToString(time_t timestamp)
Convert timestamp to a string.
static XRootDStatus CheckTPCLite(const std::string &server, uint16_t timeout=0)
static void LogHostAddresses(Log *log, uint64_t type, const std::string &hostId, std::vector< XrdNetAddr > &addresses)
Log all the addresses on the list.
static std::string NormalizeChecksum(const std::string &name, const std::string &checksum)
Normalize checksum.
static Status ProcessConfig(std::map< std::string, std::string > &config, const std::string &file)
Process a config file and return key-value pairs.
static Status ProcessConfigDir(std::map< std::string, std::string > &config, const std::string &dir)
Process a config directory and return key-value pairs.
static std::string FQDNToCC(const std::string &fqdn)
Convert the fully qualified host name to country code.
static std::string InferChecksumType(const XrdCl::URL &source, const XrdCl::URL &destination, bool zip=false)
Automatically infer the right checksum type.
static void LogPropertyList(Log *log, uint64_t topic, const char *format, const PropertyList &list)
Log property list.
static std::string Char2Hex(uint8_t *array, uint16_t size)
Print a char array as hex.
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
Definition XrdClUtils.hh:56
static Status GetHostAddresses(std::vector< XrdNetAddr > &addresses, const URL &url, AddressType type)
Resolve IP addresses.
static uint64_t GetElapsedMicroSecs(timeval start, timeval end)
Get the elapsed microseconds between two timevals.
static std::vector< std::string > GetSupportedCheckSums(const XrdCl::URL &url)
Get supported checksum types for given URL.
static AddressType String2AddressType(const std::string &addressType)
Interpret a string as address type, default to IPAll.
static int GetIntParameter(const URL &url, const std::string &name, int defaultVal)
Get a parameter either from the environment or URL.
Definition XrdClUtils.cc:81
static Status GetDirectoryEntries(std::vector< std::string > &entries, const std::string &path)
Get directory entries.
static XRootDStatus GetLocalCheckSum(std::string &checkSum, const std::string &checkSumType, const std::string &path)
Get a checksum from local file.
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
static void Trim(std::string &str)
Trim a string.
static bool CheckEC(const Message *req, const URL &url)
Check if this client can support given EC redirect.
static XRootDStatus GetRemoteCheckSum(std::string &checkSum, const std::string &checkSumType, const URL &url)
Get a checksum from a remote xrootd server.
static std::string GetStringParameter(const URL &url, const std::string &name, const std::string &defaultVal)
Get a parameter either from the environment or URL.
static XRootDStatus CheckTPC(const std::string &server, uint16_t timeout=0)
Check if peer supports tpc.
AddressType
Address type.
Definition XrdClUtils.hh:87
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
An interface for metadata redirectors.
virtual std::vector< std::string > GetSupportedCheckSums() const =0
const std::string & GetErrorMessage() const
Get error message.
void SetErrorMessage(const std::string &message)
Set the error message.
std::string ToStr() const
Convert to string.
bool isMapped() const
bool isIPType(IPType ipType) const
static const char * GetAddrs(const char *hSpec, XrdNetAddr *aListP[], int &aListN, AddrOpts opts=allIPMap, int pNum=PortInSpec)
const uint16_t suPartial
const uint16_t errInvalidAddr
const uint16_t stFatal
Fatal error, it's still an error.
const uint16_t stError
An error occurred that could potentially be retried.
const uint16_t errInternal
Internal error.
const uint16_t stOK
Everything went OK.
const int DefaultIPNoShuffle
const uint16_t errConfig
System misconfigured.
const uint16_t errOSError
const uint64_t UtilityMsg
const uint16_t errInvalidResponse
std::vector< ChunkInfo > ChunkList
List of chunks.
const uint16_t errNotSupported
const int DefaultPreferIPv4
const uint16_t errCheckSumError
Response NullRef< Response >::value
const int DefaultZipMtlnCksum
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ Config
Query server configuration.
@ Checksum
Query file checksum.
Procedure execution status.
uint16_t status
Status of the execution.
bool IsOK() const
We're fine.
std::string ToString() const
Create a string representation.