XRootD
Loading...
Searching...
No Matches
XrdClHttpPosix.cc
Go to the documentation of this file.
1
4
5#include <stdlib.h>
6#include <unistd.h>
7#include <sys/types.h>
8
10
12#include "XrdCl/XrdClStatus.hh"
14#include "XrdCl/XrdClURL.hh"
15
16#include "auth/davixx509cred.hpp"
17#include "auth/davixauth.hpp"
18
19#include <string>
20
21namespace {
22
23std::vector<std::string> SplitString(const std::string& input,
24 const std::string& delimiter) {
25 size_t start = 0;
26 size_t end = 0;
27 size_t length = 0;
28
29 auto result = std::vector<std::string>{};
30
31 do {
32 end = input.find(delimiter, start);
33
34 if (end != std::string::npos)
35 length = end - start;
36 else
37 length = input.length() - start;
38
39 if (length) result.push_back(input.substr(start, length));
40
41 start = end + delimiter.size();
42 } while (end != std::string::npos);
43
44 return result;
45}
46
47void SetTimeout(Davix::RequestParams& params, uint16_t timeout) {
48/*
49 * At NERSC archive portal, we get error when setOperationTimeout()
50 *
51 if (timeout != 0) {
52 struct timespec ts = {timeout, 0};
53 params.setOperationTimeout(&ts);
54 }
55*/
56
57 struct timespec ts = {0, 0};
58 ts.tv_sec = 30;
59 params.setConnectionTimeout(&ts);
60
61 params.setOperationRetry(0);
62 params.setOperationRetryDelay(2);
63}
64
65XrdCl::XRootDStatus FillStatInfo(const struct stat& stats, XrdCl::StatInfo* stat_info) {
66 std::ostringstream data;
67 if (S_ISDIR(stats.st_mode)) {
68 data << stats.st_dev << " " << stats.st_size << " "
71 << " " << stats.st_mtime;
72 }
73 else {
74 if (getenv("AWS_ACCESS_KEY_ID")) {
75 data << stats.st_dev << " " << stats.st_size << " "
76 << XrdCl::StatInfo::Flags::IsReadable << " " << stats.st_mtime;
77 }
78 else {
79 data << stats.st_dev << " " << stats.st_size << " "
80 << stats.st_mode << " " << stats.st_mtime;
81 }
82
83 }
84
85 if (!stat_info->ParseServerResponse(data.str().c_str())) {
87 }
88
89 return XrdCl::XRootDStatus();
90}
91
92// return NULL if no X509 proxy is found
93//Davix::X509Credential* LoadX509UserCredential() {
94// std::string myX509proxyFile;
95// if (getenv("X509_USER_PROXY") != NULL)
96// myX509proxyFile = getenv("X509_USER_PROXY");
97// else
98// myX509proxyFile = "/tmp/x509up_u" + std::to_string(geteuid());
99//
100// struct stat myX509proxyStat;
101// Davix::X509Credential* myX509proxy = NULL;
102// if (stat(myX509proxyFile.c_str(), &myX509proxyStat) == 0) {
103// myX509proxy = new Davix::X509Credential();
104// myX509proxy->loadFromFilePEM(myX509proxyFile.c_str(), myX509proxyFile.c_str(), "", NULL);
105// }
106// return myX509proxy;
107//}
108
109// see auth/davixauth.hpp
110int LoadX509UserCredentialCallBack(void *userdata,
111 const Davix::SessionInfo &info,
112 Davix::X509Credential *cert,
113 Davix::DavixError **err) {
114 std::string myX509proxyFile;
115 if (getenv("X509_USER_PROXY") != NULL)
116 myX509proxyFile = getenv("X509_USER_PROXY");
117 else
118 myX509proxyFile = "/tmp/x509up_u" + std::to_string(geteuid());
119
120 struct stat myX509proxyStat;
121 if (stat(myX509proxyFile.c_str(), &myX509proxyStat) == 0)
122 return cert->loadFromFilePEM(myX509proxyFile.c_str(), myX509proxyFile.c_str(), "", err);
123 else
124 return 1;
125}
126
127void SetX509(Davix::RequestParams& params) {
128 params.setClientCertCallbackX509(&LoadX509UserCredentialCallBack, NULL);
129
130 //Davix::X509Credential* myX509proxy = LoadX509UserCredential();
131 //if (myX509proxy != NULL) {
132 // params.setClientCertX509(*myX509proxy);
133 // delete myX509proxy;
134 //}
135
136 if (getenv("X509_CERT_DIR") != NULL)
137 params.addCertificateAuthorityPath(getenv("X509_CERT_DIR"));
138 else
139 params.addCertificateAuthorityPath("/etc/grid-security/certificates");
140}
141
142void SetAuthS3(Davix::RequestParams& params) {
143 //Davix::setLogScope(DAVIX_LOG_SCOPE_ALL);
144 //Davix::setLogScope(DAVIX_LOG_HEADER | DAVIX_LOG_S3);
145 //Davix::setLogLevel(DAVIX_LOG_TRACE);
146 params.setProtocol(Davix::RequestProtocol::AwsS3);
147 params.setAwsAuthorizationKeys(getenv("AWS_SECRET_ACCESS_KEY"),
148 getenv("AWS_ACCESS_KEY_ID"));
149 params.setAwsAlternate(true);
150 // if AWS region is not set, Davix will use the old AWS signature v2
151 if (getenv("AWS_REGION"))
152 params.setAwsRegion(getenv("AWS_REGION"));
153 else if (! getenv("AWS_SIGNATURE_V2"))
154 params.setAwsRegion("mars");
155}
156
157void SetAuthz(Davix::RequestParams& params) {
158 if (getenv("AWS_ACCESS_KEY_ID") && getenv("AWS_SECRET_ACCESS_KEY"))
159 SetAuthS3(params);
160 else
161 SetX509(params);
162}
163
164std::string SanitizedURL(const std::string& url) {
165 XrdCl::URL xurl(url);
166 std::string path = xurl.GetPath();
167 if (path.find("/") != 0) path = "/" + path;
168 std::string returl = xurl.GetProtocol() + "://"
169 + xurl.GetHostName() + ":"
170 + std::to_string(xurl.GetPort())
171 + path;
172 // for s3 storage using AWS_ACCESS_KEY_ID, filter out all CGIs
173 // Known issues:
174 // Google cloud storage does not like ?xrd.gsiusrpxy=/tmp/..., Will fail Stat()
175 if (! getenv("AWS_ACCESS_KEY_ID") && ! xurl.GetParamsAsString().empty()) {
176 returl = returl + xurl.GetParamsAsString();
177 }
178 return returl;
179}
180
181// check davix/include/davix/status/davixstatusrequest.hpp and
182// XProtocol/XProtocol.hh (XErrorCode) for corresponding error codes.
183std::pair<uint16_t, XErrorCode> ErrCodeConvert(Davix::StatusCode::Code code) {
184 if (code == Davix::StatusCode::FileNotFound)
185 return std::make_pair(XrdCl::errErrorResponse, kXR_NotFound);
186 else if (code == Davix::StatusCode::FileExist)
187 return std::make_pair(XrdCl::errErrorResponse, kXR_ItExists);
188 else if (code == Davix::StatusCode::PermissionRefused)
189 return std::make_pair(XrdCl::errErrorResponse, kXR_NotAuthorized);
190 else
191 return std::make_pair(XrdCl::errErrorResponse, kXR_InvalidRequest);
192}
193
194} // namespace
195
196namespace Posix {
197
198using namespace XrdCl;
199
200std::pair<DAVIX_FD*, XRootDStatus> Open(Davix::DavPosix& davix_client,
201 const std::string& url, int flags,
202 uint16_t timeout) {
203 Davix::RequestParams params;
204 SetTimeout(params, timeout);
205 SetAuthz(params);
206 Davix::DavixError* err = nullptr;
207 DAVIX_FD* fd = davix_client.open(&params, SanitizedURL(url), flags, &err);
208 XRootDStatus status;
209 if (!fd) {
210 auto res = ErrCodeConvert(err->getStatus());
211 status = XRootDStatus(stError, res.first, res.second, err->getErrMsg());
212 delete err;
213 }
214 else {
215 status = XRootDStatus();
216 }
217 return std::make_pair(fd, status);
218}
219
220XRootDStatus Close(Davix::DavPosix& davix_client, DAVIX_FD* fd) {
221 Davix::DavixError* err = nullptr;
222 if (davix_client.close(fd, &err)) {
223 auto errStatus =
224 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
225 delete err;
226 return errStatus;
227 }
228
229 return XRootDStatus();
230}
231
232XRootDStatus MkDir(Davix::DavPosix& davix_client, const std::string& path,
234 uint16_t timeout) {
235
236 return XRootDStatus();
237
238 Davix::RequestParams params;
239 SetTimeout(params, timeout);
240 SetAuthz(params);
241
242 auto DoMkDir = [&davix_client, &params](const std::string& path) {
243 Davix::DavixError* err = nullptr;
244 if (davix_client.mkdir(&params, SanitizedURL(path), S_IRWXU, &err) &&
245 (err->getStatus() != Davix::StatusCode::FileExist)) {
246 auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(),
247 err->getErrMsg());
248 delete err;
249 return errStatus;
250 } else {
251 return XRootDStatus();
252 }
253 };
254
255 auto url = XrdCl::URL(path);
256
257 if (flags & XrdCl::MkDirFlags::MakePath) {
258 // Also create intermediate directories
259
260 auto dirs = SplitString(url.GetPath(), "/");
261
262 std::string dirs_cumul;
263 for (const auto& d : dirs) {
264 dirs_cumul += "/" + d;
265 url.SetPath(dirs_cumul);
266 auto status = DoMkDir(url.GetLocation());
267 if (status.IsError()) {
268 return status;
269 }
270 }
271 } else {
272 // Only create final directory
273 auto status = DoMkDir(url.GetURL());
274 if (status.IsError()) {
275 return status;
276 }
277 }
278
279 return XRootDStatus();
280}
281
282XRootDStatus RmDir(Davix::DavPosix& davix_client, const std::string& path,
283 uint16_t timeout) {
284 Davix::RequestParams params;
285 SetTimeout(params, timeout);
286 SetAuthz(params);
287
288 Davix::DavixError* err = nullptr;
289 if (davix_client.rmdir(&params, path, &err)) {
290 auto errStatus =
291 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
292 delete err;
293 return errStatus;
294 }
295
296 return XRootDStatus();
297}
298
299std::pair<XrdCl::DirectoryList*, XrdCl::XRootDStatus> DirList(
300 Davix::DavPosix& davix_client, const std::string& path, bool details,
301 bool /*recursive*/, uint16_t timeout) {
302 Davix::RequestParams params;
303 SetTimeout(params, timeout);
304 SetAuthz(params);
305
306 auto dir_list = new DirectoryList();
307
308 Davix::DavixError* err = nullptr;
309
310 auto dir_fd = davix_client.opendirpp(&params, SanitizedURL(path), &err);
311 if (!dir_fd) {
312 auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(),
313 err->getErrMsg());
314 delete err;
315 return std::make_pair(nullptr, errStatus);
316 }
317
318 struct stat info;
319 while (auto entry = davix_client.readdirpp(dir_fd, &info, &err)) {
320 if (err) {
321 auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(),
322 err->getErrMsg());
323 delete err;
324 return std::make_pair(nullptr, errStatus);
325 }
326
327 StatInfo* stat_info = nullptr;
328 if (details) {
329 stat_info = new StatInfo();
330 auto res = FillStatInfo(info, stat_info);
331 if (res.IsError()) {
332 delete entry;
333 delete stat_info;
334 return std::make_pair(nullptr, res);
335 }
336 }
337
338 auto list_entry = new DirectoryList::ListEntry(path, entry->d_name, stat_info);
339 dir_list->Add(list_entry);
340
341 // do not delete "entry". davix_client.readdirpp() always return the same address
342 // and will set it to NULL when there is no more directory entry to return
343 //delete entry;
344 }
345
346 if (davix_client.closedirpp(dir_fd, &err)) {
347 auto errStatus = XRootDStatus(stError, errInternal, err->getStatus(),
348 err->getErrMsg());
349 delete err;
350 return std::make_pair(nullptr, errStatus);
351 }
352
353 return std::make_pair(dir_list, XRootDStatus());
354}
355
356XRootDStatus Rename(Davix::DavPosix& davix_client, const std::string& source,
357 const std::string& dest, uint16_t timeout) {
358
359 // most s3 storage systems either:
360 // 1. do not support rename, especially for files that were uploaded using multi-part
361 // 2. support by copy-n-delete.
362 // we could implement copy-n-delete if necessary
363 if (getenv("AWS_ACCESS_KEY_ID"))
365
366 Davix::RequestParams params;
367 SetTimeout(params, timeout);
368 SetAuthz(params);
369
370 Davix::DavixError* err = nullptr;
371 if (davix_client.rename(&params, SanitizedURL(source), SanitizedURL(dest), &err)) {
372 auto errStatus =
373 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
374 delete err;
375 return errStatus;
376 }
377
378 return XRootDStatus();
379}
380
381XRootDStatus Stat(Davix::DavPosix& davix_client, const std::string& url,
382 uint16_t timeout, StatInfo* stat_info) {
383 Davix::RequestParams params;
384 SetTimeout(params, timeout);
385 SetAuthz(params);
386
387 struct stat stats;
388 Davix::DavixError* err = nullptr;
389 if (davix_client.stat(&params, SanitizedURL(url), &stats, &err)) {
390 auto res = ErrCodeConvert(err->getStatus());
391 auto errStatus =
392 XRootDStatus(stError, res.first, res.second, err->getErrMsg());
393 delete err;
394 return errStatus;
395 }
396
397 auto res = FillStatInfo(stats, stat_info);
398 if (res.IsError()) {
399 return res;
400 }
401
402 return XRootDStatus();
403}
404
405XRootDStatus Unlink(Davix::DavPosix& davix_client, const std::string& url,
406 uint16_t timeout) {
407 Davix::RequestParams params;
408 SetTimeout(params, timeout);
409 SetAuthz(params);
410
411 Davix::DavixError* err = nullptr;
412 if (davix_client.unlink(&params, SanitizedURL(url), &err)) {
413 auto errStatus =
414 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
415 delete err;
416 return errStatus;
417 }
418
419 return XRootDStatus();
420}
421
422std::pair<int, XRootDStatus> _PRead(Davix::DavPosix& davix_client, DAVIX_FD* fd,
423 void* buffer, uint32_t size,
424 uint64_t offset, bool no_pread = false) {
425 Davix::DavixError* err = nullptr;
426 int num_bytes_read;
427 if (no_pread) { // continue reading from the current offset position
428 num_bytes_read = davix_client.read(fd, buffer, size, &err);
429 }
430 else {
431 num_bytes_read = davix_client.pread(fd, buffer, size, offset, &err);
432 }
433 if (num_bytes_read < 0) {
434 auto errStatus =
435 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
436 delete err;
437 return std::make_pair(num_bytes_read, errStatus);
438 }
439
440 return std::make_pair(num_bytes_read, XRootDStatus());
441}
442
443std::pair<int, XRootDStatus> Read(Davix::DavPosix& davix_client, DAVIX_FD* fd,
444 void* buffer, uint32_t size) {
445 return _PRead(davix_client, fd, buffer, size, 0, true);
446}
447
448std::pair<int, XRootDStatus> PRead(Davix::DavPosix& davix_client, DAVIX_FD* fd,
449 void* buffer, uint32_t size, uint64_t offset) {
450 return _PRead(davix_client, fd, buffer, size, offset, false);
451}
452
453std::pair<int, XrdCl::XRootDStatus> PReadVec(Davix::DavPosix& davix_client,
454 DAVIX_FD* fd,
455 const XrdCl::ChunkList& chunks,
456 void* buffer) {
457 const auto num_chunks = chunks.size();
458 std::vector<Davix::DavIOVecInput> input_vector(num_chunks);
459 std::vector<Davix::DavIOVecOuput> output_vector(num_chunks);
460
461 for (size_t i = 0; i < num_chunks; ++i) {
462 input_vector[i].diov_offset = chunks[i].offset;
463 input_vector[i].diov_size = chunks[i].length;
464 input_vector[i].diov_buffer = chunks[i].buffer;
465 }
466
467 Davix::DavixError* err = nullptr;
468 int num_bytes_read = davix_client.preadVec(
469 fd, input_vector.data(), output_vector.data(), num_chunks, &err);
470 if (num_bytes_read < 0) {
471 auto errStatus =
472 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
473 delete err;
474 return std::make_pair(num_bytes_read, XRootDStatus(stError, errUnknown));
475 }
476
477 return std::make_pair(num_bytes_read, XRootDStatus());
478}
479
480std::pair<int, XrdCl::XRootDStatus> PWrite(Davix::DavPosix& davix_client,
481 DAVIX_FD* fd, uint64_t offset,
482 uint32_t size, const void* buffer,
483 uint16_t timeout) {
484 Davix::DavixError* err = nullptr;
485 off_t new_offset = davix_client.lseek(fd, offset, SEEK_SET, &err);
486 if (uint64_t(new_offset) != offset) {
487 auto errStatus =
488 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
489 delete err;
490 return std::make_pair(new_offset, errStatus);
491 }
492 int num_bytes_written = davix_client.write(fd, buffer, size, &err);
493 if (num_bytes_written < 0) {
494 auto errStatus =
495 XRootDStatus(stError, errInternal, err->getStatus(), err->getErrMsg());
496 delete err;
497 return std::make_pair(num_bytes_written, errStatus);
498 }
499
500 return std::make_pair(num_bytes_written, XRootDStatus());
501}
502
503} // namespace Posix
@ kXR_InvalidRequest
Definition XProtocol.hh:996
@ kXR_ItExists
@ kXR_NotAuthorized
@ kXR_NotFound
@ kXR_Unsupported
struct stat Stat
Definition XrdCks.cc:49
XRootDStatus DoMkDir(FileSystem *fs, Env *env, const FSExecutor::CommandParams &args)
Definition XrdClFS.cc:496
#define stat(a, b)
Definition XrdPosix.hh:96
Object stat info.
bool ParseServerResponse(const char *data)
Parse server response and fill up the object.
@ IsReadable
Read access is allowed.
@ IsDir
This is a directory.
@ XBitSet
Executable/searchable bit set.
@ IsWritable
Write access is allowed.
URL representation.
Definition XrdClURL.hh:31
std::pair< int, XrdCl::XRootDStatus > PReadVec(Davix::DavPosix &davix_client, DAVIX_FD *fd, const XrdCl::ChunkList &chunks, void *buffer)
std::pair< int, XrdCl::XRootDStatus > PWrite(Davix::DavPosix &davix_client, DAVIX_FD *fd, uint64_t offset, uint32_t size, const void *buffer, uint16_t timeout)
std::pair< int, XRootDStatus > PRead(Davix::DavPosix &davix_client, DAVIX_FD *fd, void *buffer, uint32_t size, uint64_t offset)
std::pair< int, XRootDStatus > _PRead(Davix::DavPosix &davix_client, DAVIX_FD *fd, void *buffer, uint32_t size, uint64_t offset, bool no_pread=false)
XRootDStatus Unlink(Davix::DavPosix &davix_client, const std::string &url, uint16_t timeout)
XRootDStatus Rename(Davix::DavPosix &davix_client, const std::string &source, const std::string &dest, uint16_t timeout)
const uint16_t errUnknown
Unknown error.
ReadImpl< false > Read(Ctx< File > file, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void * > buffer, uint16_t timeout=0)
Factory for creating ReadImpl objects.
const uint16_t errErrorResponse
MkDirImpl< false > MkDir
const uint16_t stError
An error occurred that could potentially be retried.
CloseImpl< false > Close(Ctx< File > file, uint16_t timeout=0)
Factory for creating CloseImpl objects.
const uint16_t errDataError
data is corrupted
const uint16_t errInternal
Internal error.
std::vector< ChunkInfo > ChunkList
List of chunks.
RmDirImpl< false > RmDir
OpenImpl< false > Open(Ctx< File > file, Arg< std::string > url, Arg< OpenFlags::Flags > flags, Arg< Access::Mode > mode=Access::None, uint16_t timeout=0)
Factory for creating ReadImpl objects.
DirListImpl< false > DirList
@ MakePath
create the entire directory tree if it doesn't exist