#include "QFramework/TQPathManager.h"
#include <algorithm>
#include <cstdio>
#include <string>
#include <vector>
#include <filesystem>
#include "QFramework/TQUtils.h"
#include "QFramework/TQStringUtils.h"
#include "QFramework/TQLibrary.h"
#include "TString.h"
#include "TSystem.h"
ClassImp(TQPathManager)
const bool TQPathManager::gUseTemporaryStorageDefault =
TQUtils::readEnvVarValueBool("CAF_USE_TEMP_DIR",
TQUtils::readEnvVarValueBool("CAF_USE_LOCAL_STORAGE", true)
);
int TQPathManager::filenameIndex = 0;
TQPathManager TQPathManager::gPathManager;
bool TQPathManager::gPathManagerExists = false;
std::string TQPathManager::findFileFromEnvVar(const TString& filename, const std::string& envVar, bool printWarnings){
return findFileFromEnvVar_internal(filename, envVar, printWarnings, true);
}
std::string TQPathManager::findFileFromEnvVarWithoutExecDir(const TString& filename, const std::string& envVar, bool printWarnings){
return findFileFromEnvVar_internal(filename, envVar, printWarnings, false);
}
std::string TQPathManager::findFileFromEnvVar_internal(const TString& filename, const std::string& envVar, bool printWarnings, bool searchExecutionDir){
std::string searchPathList = TQUtils::readEnvVarValue(envVar);
if (searchPathList == ""){
DEBUGclass("Calling findFileFromList with an empty searchPathList since the environment variable %s was not found.", envVar.c_str());
}
std::string filePath = TQPathManager::findFileFromList_internal(filename, searchPathList, printWarnings, searchExecutionDir);
if ((searchPathList == "") && (filePath == "")){
TString message = TQStringUtils::format("The environment variable %s is not defined. Is this the reason why the file %s could not be found?", envVar.c_str(), filename.Data());
if (printWarnings) {WARNclass(message);}
else {DEBUGclass(message);}
}
return filePath;
}
std::string TQPathManager::findFileFromList(TString filename, const std::string& searchPathList, bool printWarnings){
return findFileFromList_internal(filename, searchPathList, printWarnings, true);
}
std::string TQPathManager::findFileFromListWithoutExecDir(TString filename, const std::string& searchPathList, bool printWarnings){
return findFileFromList_internal(filename, searchPathList, printWarnings, false);
}
void TQPathManager::setWorkingDirectory(TString dir){
DEBUGclass("working directory changed from '%s' to '%s'",this->pwd.Data(),dir.Data());
this->pwd = dir;
}
std::string TQPathManager::findFileFromList_internal(TString filename, const std::string& searchPathList, bool printWarnings, bool searchExecutionDir){
filename = TQStringUtils::trim(filename);
if (searchPathList == "") {DEBUGclass("Trying to find %s locally.", filename.Data());}
else {DEBUGclass("Trying to find %s in %s.", filename.Data(), searchPathList.c_str());}
if (filename == ""){
DEBUGclass("The passed argument filename is empty. Exiting the method.");
return "";
}
TString localDir = gSystem->pwd();
TString localPath = TQFolder::concatPaths(localDir, filename);
if (gSystem->IsAbsoluteFileName(filename.Data())){
if (TQUtils::fileExists(filename)){
std::string totalPath = filename.Data();
DEBUGclass("Found absolute path: %s. Not resolving.", filename.Data());
return totalPath;
}
TString message = TQStringUtils::format("It looks like %s is an absolute path, but there is no such file (is it maybe a directory?). Exiting this method.", filename.Data());
if (printWarnings) {WARNclass(message.Data());}
else {DEBUGclass(message.Data());}
return "";
}
if ((searchExecutionDir) && (TQUtils::fileExists(localPath))){
std::string totalPath = localPath.Data();
DEBUGclass("Found %s here: %s", filename.Data(), totalPath.c_str());
return totalPath;
}
std::vector< TString > searchPathVector;
searchPathVector = TQStringUtils::split(searchPathList, ":");
for (auto& searchPath:searchPathVector){
TString filePath = TQFolder::concatPaths(searchPath, filename);
if (TQUtils::fileExists(filePath)){
std::string totalPath = filePath.Data();
DEBUGclass("Found %s here: %s", filename.Data(), totalPath.c_str());
return totalPath;
}
}
TString message = TQStringUtils::format("The file %s could not be found.", filename.Data());
if (printWarnings) {WARNclass(message.Data());}
else {DEBUGclass(message.Data());}
return "";
}
TQPathManager* TQPathManager::getPathManager(){
if (TQPathManager::gPathManagerExists)
return &gPathManager;
return nullptr;
}
bool TQPathManager::getUseTemporaryStorage() const {
return this->useTemporaryStorage;
}
void TQPathManager::setUseTemporaryStorage(bool useTemporaryStorage){
if ((this->pathsRequested) and (this->useTemporaryStorage != useTemporaryStorage))
WARNclass("Changing the variable useTemporaryStorage after path names have been requested might cause problems. Especially the method \"TQPathManager::deleteLocalFile(...)\" will not work reliably. Try to avoid calling this method late during runtime.");
this->useTemporaryStorage = useTemporaryStorage;
}
TString TQPathManager::getLocalDirectory() const {
return this->localDirectory;
}
bool TQPathManager::setLocalDirectory(TString localDirectory){
if (!localDirectory.EndsWith("/"))
localDirectory.Append("/");
if ((this->pathsRequested) and (this->localDirectory != localDirectory))
WARNclass("Changing the variable localDirectory after path names have been requested might cause problems. Especially the method \"TQPathManager::deleteLocalFile(...)\" will not work reliably. Try to avoid calling this method late during runtime.");
if (!TQUtils::ensureDirectory(localDirectory)){
WARNclass("Local directory %s could not be created. Do you have the necessary rights? Not setting localDirectory.", localDirectory.Data());
return false;
}
this->localDirectory = localDirectory;
if (this->useTemporaryStorage){
DEBUGclass("TQPathManager uses the local directory %s for temporary storage.", localDirectory.Data());
}
return true;
}
void TQPathManager::setVerbose(bool verbose){
this->verbose = verbose;
}
void TQPathManager::registerLocalPath(TString targetFile){
targetFile = TQStringUtils::trim(targetFile);
if (!isInLocalPaths_target(targetFile)){
WARNclass("You request that even getTargetPath(%s) return a local path. Please call this method only after you first retrieve this local path. Otherwise it has no effect.", targetFile.Data());
return;
}
alwaysReturnLocalPath[targetFile] = true;
}
void TQPathManager::unregisterLocalPath(TString targetFile){
targetFile = TQStringUtils::trim(targetFile);
if (!isInLocalPaths_target(targetFile)){
WARNclass("You request that getTargetPath(%s) not return a local path. Since no local path has ever been created, there should be no reason to call this method. Please check your code.", targetFile.Data());
return;
}
if (!alwaysReturnLocalPath[targetFile]){
DEBUGclass("alwaysReturnLocalPath[%s] is already false", targetFile.Data());
return;
}
alwaysReturnLocalPath[targetFile] = false;
}
std::string TQPathManager::getTargetPath(TString targetFile, bool printWarnings){
gSystem->ExpandPathName(targetFile);
targetFile = TQStringUtils::trim(targetFile);
if (isInLocalPaths_target(targetFile)){
if (alwaysReturnLocalPath[targetFile]){
DEBUGclass("Returning local path, because alwaysReturnLocalPath[%s] == true", targetFile.Data());
return getLocalPath(targetFile);
}
else if (printWarnings){
WARNclass("You are requesting the target path for a file that is already assigned temporary storage. Make sure that you don't copy files over.");
}
}
TString targetPath = getTargetPathName(targetFile);
DEBUGclass("Target path found: %s.", targetPath.Data());
if (targetPath.BeginsWith("//")){
ERRORclass("Congratulations: you just found a bug! Please post the following lines starting with ERROR on this website: https://gitlab.cern.ch/atlas-caf/CAFCore/issues/54. If possible, please also tell us your current commit of CAFCore (find it out by going into your CAFCore directory and typing \"git rev-parse HEAD\").");
ERRORclass("$CAFOUTPUTDIR = %s", TQUtils::readEnvVarValue("CAFOUTPUTDIR").c_str());
ERRORclass("this->pwd = %s", this->pwd.Data());
ERRORclass("gSystem->pwd() = %s", gSystem->pwd());
ERRORclass("$PWD = %s", TQUtils::readEnvVarValue("PWD").c_str());
ERRORclass("targetPath = %s", targetPath.Data());
ERRORclass("this->useTemporaryStorage = %s", this->useTemporaryStorage ? "true" : "false");
ERRORclass("this->filenameIndex = %i", this->filenameIndex);
ERRORclass("this->pathsRequested = %s", this->pathsRequested ? "true" : "false");
targetPath = targetPath.Replace(0, 1, "", 0);
ERRORclass("Trying to copy your output file to %s. If this does not look correct, please log out of the current shell and log back in to temporarily fix this error.", targetPath.Data());
}
if (!isInTargetPaths(targetPath))
targetPaths.push_back(targetPath);
this->pathsRequested = true;
return targetPath.Data();
}
std::string TQPathManager::getLocalPath(TString filename){
if (!this->useTemporaryStorage){
DEBUGclass("You have temporary storage disabled. Return target path instead of local path.");
return this->getTargetPath(filename);
}
filename = TQStringUtils::trim(filename);
if (this->isInLocalPaths_target(filename)){
DEBUGclass("A local file for %s does already exist at %s.", filename.Data(), localPaths.at(filename).Data());
return localPaths.at(filename).Data();
}
if (this->isInTargetPaths(this->getTargetPathName(filename))){
WARNclass("You are requesting a local path for a file that a target path has been requested for. This file might copy another one over.");
}
if (this->isInLocalPaths_local(TQFolder::concatPaths(this->localDirectory, filename))){
WARNclass("You are trying to get a local file for a local file. This doesn't make much sense. Continuing anyway.");
}
int extensionPos = TQStringUtils::rfind(filename,".");
TString extension = "";
if (extensionPos != kNPOS) extension = filename(extensionPos,filename.Length());
TString localFilename = std::to_string(::getpid()) + "_" + std::to_string(TQPathManager::filenameIndex++) + "_" + std::to_string(filename.Hash()) + extension;
TString localPath = TQFolder::concatPaths(this->localDirectory, localFilename);
if (!TQUtils::ensureDirectoryForFile(localPath)){
WARNclass("Cannot create directory for local file %s. Use global storage instead and return target path.", localPath.Data());
return this->getTargetPath(filename);
}
localPaths[filename] = localPath;
alwaysReturnLocalPath[filename] = false;
DEBUGclass("Assigned local file %s to target file %s.", localPath.Data(), filename.Data());
this->pathsRequested = true;
return localPath.Data();
}
std::string TQPathManager::findConfigPath(TString filename, bool errorMessageIfNotFound, bool searchExecutionDir){
if (filename == "") {
WARNclass("Cannot find config from an empty string! Returning empty string.");
return "";
}
std::vector<TString> splittedFileName = TQStringUtils::split(filename, ".root:");
TString filenameBase;
TString stringTail = "";
if (splittedFileName.size() == 2) {
filenameBase = TQStringUtils::concat(splittedFileName[0],".root");
stringTail = TQStringUtils::concat(":", splittedFileName[1]);
} else if (splittedFileName.size() ==1) {
filenameBase = filename;
} else {
ERRORclass("Something odd happened. You are trying to find the path for the config file '%s' and is seems to have two times the identifier '.root:' in it. This is not supported, please check your code!", filename.Data());
}
TString filePath = TQPathManager::findFileFromEnvVar_internal(filenameBase, "CAFANALYSISSHARE", true, searchExecutionDir);
if (filePath == "" && errorMessageIfNotFound && TQUtils::readEnvVarValue("CAFANALYSISSHARE") == "") {
WARNclass("Did you forget to set the environment variable 'CAFANALYSISSHARE' in your analysis package?");
}
TString returnFileName = filePath+stringTail;
return returnFileName.Data();
}
bool TQPathManager::deleteLocalFile(TString filename){
filename = TQStringUtils::trim(filename);
TString pathToDelete = "";
if (!this->useTemporaryStorage){
DEBUGclass("You have temporary storage disabled. Deleting target file instead of local file.");
pathToDelete = this->getTargetPathName(filename);
if (isInTargetPaths(pathToDelete))
this->deleteTargetPathEntry(pathToDelete);
}
else{
if (!this->isInLocalPaths_target(filename)){
DEBUGclass("The file %s does not have a local file. Cannot delete it.", filename.Data());
return false;
}
pathToDelete = localPaths.at(filename);
localPaths.erase(filename);
alwaysReturnLocalPath.erase(filename);
DEBUGclass("Deleted the database entry corresponding to %s.", filename.Data());
}
if (TQUtils::fileExists(pathToDelete)){
remove(pathToDelete.Data());
if (!this->useTemporaryStorage)
DEBUGclass("Deleted the file %s", pathToDelete.Data());
else{
DEBUGclass("Deleted the local copy of %s.", filename.Data());
}
}
else{
DEBUGclass("Nothing to delete. The file %s does not exist.", pathToDelete);
}
return true;
}
TQPathManager::TQPathManager(bool useTemporaryStorage){
TString tempDir = TQUtils::readEnvVarValue("CAF_TEMP_DIR").c_str();
if (tempDir.Length()==0) {
TString username = TQUtils::readEnvVarValue("USER").c_str();
if (username == ""){
username = "CAF_temporary";
}
tempDir = TQUtils::readEnvVarValue("TMP");
if (tempDir.Length()==0) {
tempDir = "/tmp/";
}
tempDir = TQFolder::concatPaths(tempDir, username);
}
TQStringUtils::ensureTrailingText(tempDir,"/");
this->initialize(tempDir, useTemporaryStorage);
}
TQPathManager::TQPathManager(TString localDirectory, bool useTemporaryStorage){
this->initialize(localDirectory, useTemporaryStorage);
}
TQPathManager::~TQPathManager(){
DEBUGclass("Started Destructor");
if (localPaths.size() > 0)
INFOclass("Copying files from local storage to target location: %s.", this->determineTargetDir().Data());
for (const auto& it:localPaths){
TString targetPath = getTargetPathName(it.first);
if (targetPath.EqualTo("")){
WARNclass("Couldn't create target path. Not copying files.");
break;
}
TString localPath = it.second;
if (TQUtils::fileExists(localPath)){
DEBUGclass("Copying %s to %s.", localPath.Data(), targetPath.Data());
if (verbose) INFO("Writing file to %s.", targetPath.Data());
if (!TQUtils::ensureDirectoryForFile(targetPath)){
WARNclass("Cannot create the directory for the file %s. Trying to copy anyway.", targetPath.Data());
}
if (TQUtils::fileExists(targetPath)){
DEBUGclass("There is already a file at %s. Copying it over.", targetPath.Data());
remove(targetPath.Data());
}
try {
std::filesystem::rename(localPath.Data(), targetPath.Data());
DEBUGclass("File %s was be renamed to %s.", localPath.Data(), targetPath.Data());
} catch (const std::filesystem::filesystem_error& ex) {
DEBUGclass("File %s could not be renamed to %s. Copying instead.\nError: %s", localPath.Data(), targetPath.Data(), ex.what());
try {
std::filesystem::copy(localPath.Data(), targetPath.Data());
std::filesystem::remove(localPath.Data());
DEBUGclass("File %s was be copied to %s.", localPath.Data(), targetPath.Data());
} catch (const std::filesystem::filesystem_error& ex) {
ERRORclass("Could not rename or copy file %s to %s.\nError: %s", localPath.Data(), targetPath.Data(), ex.what());
}
}
}
else{
WARNclass("You used \"TQPathManager::getLocalPath()\" to request a filename, but it seems that you did not create the file. Is this a logical error in your code? The file would have been written to %s.", targetPath.Data());
DEBUGclass("Cannot copy %s to %s. The file does not seem to exist.", localPath.Data(), targetPath.Data());
}
}
if (this == &TQPathManager::gPathManager){
DEBUGclass("The static member path manager is being destroyed.");
TQPathManager::gPathManagerExists = false;
}
else{
DEBUGclass("A path manager that is not the static member is being destroyed.");
}
DEBUGclass("Completed Destructor");
}
std::vector<TString> TQPathManager::getAllTargetPaths(){
std::vector<TString> allTargetPaths;
allTargetPaths = this->targetPaths;
for (const auto& it:this->localPaths){
allTargetPaths.push_back(getTargetPathName(it.first));
}
return allTargetPaths;
}
TString TQPathManager::getTargetPathName(TString targetFile) const {
if (gSystem->IsAbsoluteFileName(targetFile.Data())){
return targetFile;
}
TString targetDirectory = this->determineTargetDir();
TString targetPath = TQFolder::concatPaths(targetDirectory, targetFile);
if (!TQUtils::ensureDirectoryForFile(targetPath)){
ERRORclass("Cannot create directory for file %s.", targetPath.Data());
}
return targetPath;
}
bool TQPathManager::isInLocalPaths_target(TString filename) const {
filename = TQStringUtils::trim(filename);
auto localPathname = localPaths.find(filename);
if(localPathname == localPaths.end()){
return false;
}
if (!localPathname->second.IsNull())
return true;
return false;
}
bool TQPathManager::isInLocalPaths_local(const TString& filePath) const {
for (const auto& it:localPaths){
if (it.second == filePath)
return true;
}
return false;
}
bool TQPathManager::isInTargetPaths(const TString& filePath) const{
if (std::find(targetPaths.begin(), targetPaths.end(), filePath) == targetPaths.end())
return false;
else
return true;
}
void TQPathManager::deleteTargetPathEntry(const TString& filePath){
if (!isInTargetPaths(filePath)){
DEBUGclass("Cannot delete target path entry %s, because it does not exist.", filePath.Data());
return;
}
std::vector<TString>::iterator it = std::find(targetPaths.begin(), targetPaths.end(), filePath);
targetPaths.erase(it);
return;
}
TString TQPathManager::determineTargetDir() const {
TString targetDirectory = TQUtils::readEnvVarValue("CAFOUTPUTDIR");
if (targetDirectory.EqualTo(""))
return this->pwd;
if (!gSystem->IsAbsoluteFileName(targetDirectory.Data())){
WARNclass("The path %s specified in the environment variable \"CAFOUTPUTDIR\" is not an absolute directory. Write output in working directory instead.", targetDirectory.Data());
return this->pwd;
}
if (!TQUtils::ensureDirectory(targetDirectory)){
WARNclass("The path %s specified in the environment variable \"CAFOUTPUTDIR\" cannot be created. Write output in working directory instead.", targetDirectory.Data());
return this->pwd;
}
return targetDirectory;
}
void TQPathManager::initialize(const TString& localDirectory, bool useTemporaryStorage){
DEBUGclass("Creating instance of TQPathManager with local directory %s%s using temporary storage.", localDirectory.Data(), useTemporaryStorage ? "" : ", but not");
this->useTemporaryStorage = useTemporaryStorage;
this->pathsRequested = false;
this->pwd = gSystem->pwd();
if ((!gSystem->IsAbsoluteFileName(this->pwd.Data())) or (!TQUtils::directoryExists(this->pwd))){
WARNclass("Current working directory could not be determined. This is not a problem if the environment variable \"CAFOUTPUTDIR\" is set.");
}
DEBUGclass("The current working directory is set to %s.", this->pwd.Data());
if (!setLocalDirectory(localDirectory)){
this->useTemporaryStorage = false;
}
this->verbose = false;
if (this == &TQPathManager::gPathManager){
DEBUG("The static member path manager is being created.");
TQPathManager::gPathManagerExists = true;
}
else{
DEBUG("A path manager that is not the static member is being created.");
}
}