diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /fastos |
Publish
Diffstat (limited to 'fastos')
76 files changed, 16635 insertions, 0 deletions
diff --git a/fastos/.gitignore b/fastos/.gitignore new file mode 100644 index 00000000000..70fbbb55a20 --- /dev/null +++ b/fastos/.gitignore @@ -0,0 +1,12 @@ +*.ilk +*.pdb +.Build_completed +.Dist_completed +.Install_completed +.PreBuild_completed +bin +include +lib +update.log +Makefile +Testing diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt new file mode 100644 index 00000000000..a8837b17066 --- /dev/null +++ b/fastos/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + LIBS + src/vespa/fastos + + TESTS + src/tests +) diff --git a/fastos/OWNERS b/fastos/OWNERS new file mode 100644 index 00000000000..c6b7fc4c94f --- /dev/null +++ b/fastos/OWNERS @@ -0,0 +1,2 @@ +balder +arnej27959 diff --git a/fastos/src/.gitignore b/fastos/src/.gitignore new file mode 100644 index 00000000000..87807fb815c --- /dev/null +++ b/fastos/src/.gitignore @@ -0,0 +1,4 @@ +/Makefile.ini +/config_command.sh +/fastos.mak +/project.dsw diff --git a/fastos/src/testlist.txt b/fastos/src/testlist.txt new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/fastos/src/testlist.txt diff --git a/fastos/src/tests/.gitignore b/fastos/src/tests/.gitignore new file mode 100644 index 00000000000..2f67e70e89d --- /dev/null +++ b/fastos/src/tests/.gitignore @@ -0,0 +1,29 @@ +/Makefile +/backtracetest +/backtracetest.log +/filetest +/filetest.log +/gmtime +/gmtime.log +/prefetchtest +/prefetchtest.log +/processtest +/processtest.log +/sockettest +/sockettest.log +/threadtest +/threadtest.log +/timetest +/timetest.log +/typetest +/typetest.log +/usecputest +fastos_backtracetest_app +fastos_filetest_app +fastos_prefetchtest_app +fastos_processtest_app +fastos_sockettest_app +fastos_threadtest_app +fastos_timetest_app +fastos_typetest_app +fastos_usecputest_app diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt new file mode 100644 index 00000000000..06e891b88b3 --- /dev/null +++ b/fastos/src/tests/CMakeLists.txt @@ -0,0 +1,64 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fastos_processtest_app + SOURCES + processtest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_processtest_app NO_VALGRIND COMMAND fastos_processtest_app fastos_processtest_app fastos_usecputest_app) +vespa_add_executable(fastos_prefetchtest_app + SOURCES + prefetchtest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_prefetchtest_app NO_VALGRIND COMMAND fastos_prefetchtest_app) +vespa_add_executable(fastos_filetest_app + SOURCES + filetest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_filetest_app NO_VALGRIND COMMAND fastos_filetest_app) +vespa_add_executable(fastos_sockettest_app + SOURCES + sockettest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_sockettest_app NO_VALGRIND COMMAND fastos_sockettest_app) +vespa_add_executable(fastos_threadtest_app + SOURCES + threadtest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_threadtest_app NO_VALGRIND COMMAND fastos_threadtest_app) +vespa_add_executable(fastos_backtracetest_app + SOURCES + backtracetest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_backtracetest_app NO_VALGRIND COMMAND fastos_backtracetest_app) +vespa_add_executable(fastos_timetest_app + SOURCES + timetest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_timetest_app NO_VALGRIND COMMAND fastos_timetest_app) +vespa_add_executable(fastos_typetest_app + SOURCES + typetest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_typetest_app NO_VALGRIND COMMAND fastos_typetest_app) +vespa_add_executable(fastos_usecputest_app + SOURCES + usecputest.cpp + DEPENDS + fastos +) +vespa_add_test(NAME fastos_usecputest_app NO_VALGRIND COMMAND fastos_usecputest_app) diff --git a/fastos/src/tests/backtracetest.cpp b/fastos/src/tests/backtracetest.cpp new file mode 100644 index 00000000000..2ad80c18422 --- /dev/null +++ b/fastos/src/tests/backtracetest.cpp @@ -0,0 +1,170 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/fastos/backtrace.h> +#include <assert.h> +#include <string.h> + +#include "tests.h" + +#if defined __x86_64__ +class Tracker +{ +private: + int _found; + int _level; + void dotrace() { + _found = FastOS_backtrace(_codepoints, _level); + } + void deepFill(); + +protected: + virtual void deepFill20(); + virtual void deepFill19(); + virtual void deepFill18(); + virtual void deepFill17(); + virtual void deepFill16(); + virtual void deepFill15(); + virtual void deepFill14(); + virtual void deepFill13(); + virtual void deepFill12(); + virtual void deepFill11(); + virtual void deepFill10(); + virtual void deepFill9(); + virtual void deepFill8(); + virtual void deepFill7(); + virtual void deepFill6(); + virtual void deepFill5(); + virtual void deepFill4(); + virtual void deepFill3(); + virtual void deepFill2(); + virtual void deepFill1(); + + virtual void deepFill0(); + +public: + void *_codepoints[25]; + Tracker() : _found(0), _level(25) {} + virtual ~Tracker() { } + void doTest(int levels) { + for (int j=0; j<25; j++) { + _codepoints[j] = (void *)0; + } + _level = levels; + deepFill(); + printf("found levels: %d\n", _found); + for (int i=0; i<levels; i++) { + printf("level %2d -> %p\n", i, _codepoints[i]); + if (_codepoints[i] == 0) break; + } + } + int found() { return _found; } +}; + +class Tracker2: public Tracker +{ +protected: + virtual void deepFill20(); + virtual void deepFill18(); + virtual void deepFill16(); + virtual void deepFill14(); + virtual void deepFill12(); + virtual void deepFill10(); + virtual void deepFill8(); + virtual void deepFill6(); + virtual void deepFill4(); + virtual void deepFill2(); +}; + + +class BackTraceTest : public BaseTest +{ + +public: + void TestBackTrace () + { + bool rc = true; + + TestHeader("backtrace test"); + + Tracker2 t; + + t.doTest(25); + Progress(rc, "minimal functionality"); + t.doTest(25); + rc = (t._codepoints[10] != 0); + Progress(rc, "many levels"); + rc = (t.found() > 10); + Progress(rc, "many levels retval"); + t.doTest(8); + rc = (t.found() == 8); + Progress(rc, "few levels retval"); + rc = (t._codepoints[8] == 0); + Progress(rc, "few levels"); + + PrintSeparator(); + } + + int Main () + { + TestBackTrace(); + return 0; + } +}; + + +int main (int argc, char **argv) +{ + BackTraceTest app; + + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} + + +void Tracker2::deepFill20() { printf("a"); deepFill19(); printf("a"); } +void Tracker2::deepFill18() { printf("c"); deepFill17(); printf("c"); } +void Tracker2::deepFill16() { printf("e"); deepFill15(); printf("e"); } +void Tracker2::deepFill14() { printf("g"); deepFill13(); printf("g"); } +void Tracker2::deepFill12() { printf("i"); deepFill11(); printf("i"); } +void Tracker2::deepFill10() { printf("k"); deepFill9(); printf("k"); } +void Tracker2::deepFill8() { printf("m"); deepFill7(); printf("m"); } +void Tracker2::deepFill6() { printf("o"); deepFill5(); printf("o"); } +void Tracker2::deepFill4() { printf("q"); deepFill3(); printf("q"); } +void Tracker2::deepFill2() { printf("s"); deepFill1(); printf("s"); } + + +void Tracker::deepFill() { deepFill20(); printf("\n"); } + +void Tracker::deepFill20() { printf("a"); deepFill19(); } +void Tracker::deepFill19() { printf("b"); deepFill18(); } +void Tracker::deepFill18() { printf("c"); deepFill17(); } +void Tracker::deepFill17() { printf("d"); deepFill16(); } +void Tracker::deepFill16() { printf("e"); deepFill15(); } +void Tracker::deepFill15() { printf("f"); deepFill14(); } +void Tracker::deepFill14() { printf("g"); deepFill13(); } +void Tracker::deepFill13() { printf("h"); deepFill12(); } +void Tracker::deepFill12() { printf("i"); deepFill11(); } +void Tracker::deepFill11() { printf("j"); deepFill10(); } +void Tracker::deepFill10() { printf("k"); deepFill9(); } +void Tracker::deepFill9() { printf("l"); deepFill8(); } +void Tracker::deepFill8() { printf("m"); deepFill7(); } +void Tracker::deepFill7() { printf("n"); deepFill6(); } +void Tracker::deepFill6() { printf("o"); deepFill5(); } +void Tracker::deepFill5() { printf("p"); deepFill4(); } +void Tracker::deepFill4() { printf("q"); deepFill3(); } +void Tracker::deepFill3() { printf("r"); deepFill2(); } +void Tracker::deepFill2() { printf("s"); deepFill1(); } +void Tracker::deepFill1() { printf("t"); deepFill0(); } + +void Tracker::deepFill0() { dotrace(); } + +#else +int +main(int argc, char **argv) +{ + (void)argc; + (void)argv; + printf("No backtrace support, skipping tests...\n"); + return 0; +} +#endif diff --git a/fastos/src/tests/coretest.cpp b/fastos/src/tests/coretest.cpp new file mode 100644 index 00000000000..a9efcc7090b --- /dev/null +++ b/fastos/src/tests/coretest.cpp @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> + + +static void +bomb(void) +{ + char *p; + + p = NULL; + *p = 4; +} + +class FastS_Bomber : public FastOS_Runnable +{ + void Run(FastOS_ThreadInterface *thread, void *arg) + { + (void) thread; + (void) arg; + bomb(); + } +}; + +static int +bombMain(void) +{ + FastOS_ThreadPool *pool = new FastOS_ThreadPool(128*1024); + FastS_Bomber bomber; + FastOS_ThreadInterface *thread; + + thread = pool->NewThread(&bomber, NULL); + if (thread != NULL) + thread->Join(); + + pool->Close(); + delete pool; + return (0); +} + + +class FastS_CoreTestApp : public FastOS_Application +{ +public: + FastS_CoreTestApp(void) { } + ~FastS_CoreTestApp(void) { } + int Main(void); +}; + + +int +FastS_CoreTestApp::Main(void) +{ + + return bombMain(); +} + + +int +main(int argc, char **argv) +{ + FastS_CoreTestApp app; + setvbuf(stdout, NULL, _IOLBF, 8192); + if (argc == 1) + return app.Entry(argc, argv); + else + return bombMain(); +} diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp new file mode 100644 index 00000000000..bc8fc0bef0c --- /dev/null +++ b/fastos/src/tests/coretest2.cpp @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + + +#include <vespa/fastos/fastos.h> + + +static void +bomb(void) +{ + char *p; + + p = NULL; + *p = 4; +} + +void *BomberRun(void *arg) +{ + (void) arg; + bomb(); + return NULL; +}; + +static int +bombMain(void) +{ + pthread_t thread; + void *ret; + + (void) pthread_create(&thread, NULL, BomberRun, NULL); + ret = NULL; + (void) pthread_join(thread, &ret); + return (0); +} + + +int +main(int argc, char **argv) +{ + (void) argc; + (void) argv; + return bombMain(); +} diff --git a/fastos/src/tests/filetest.cpp b/fastos/src/tests/filetest.cpp new file mode 100644 index 00000000000..24483076ed0 --- /dev/null +++ b/fastos/src/tests/filetest.cpp @@ -0,0 +1,907 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <memory> +#include <vespa/fastos/fastos.h> +#include "tests.h" + +namespace { + + // Create the named file, and write it's filename into it. + // Return true on success + bool createFile(const char* fileName) { + FastOS_StatInfo statInfo; + FastOS_File cf(fileName); + return ( cf.OpenWriteOnly() && + cf.CheckedWrite(fileName, strlen(fileName)) && + cf.Close() && + FastOS_File::Stat(fileName, &statInfo) && + statInfo._isRegular ); + } + + bool createFile(const char* fileName, + const unsigned int size) { + FastOS_File cf(fileName); + bool success = false; + if (cf.OpenWriteOnlyTruncate()) { + auto buf = std::make_unique<char[]>(size); // Could be dangerous.. + if (buf) { + memset(buf.get(), 0, size); // dont write uninitialized bytes since valgrind will complain + if (cf.CheckedWrite(buf.get(), size)) { + success = true; + } + } + cf.Close(); + } + return success; + } +} + + +class FileTest : public BaseTest +{ +private: + virtual bool useProcessStarter() const { return true; } +public: + static const char * roFilename; + static const char * woFilename; + static const char * rwFilename; + + void DirectoryTest() + { + TestHeader ("Directory management (remove & empty) test"); + + const char *dirName = "tmpTestDir"; + char file1[1024]; + char file2[1024]; + char file3[1024]; + char file4[1024]; + char file5[1024]; + char subdir1[1024]; + char subdir2[1024]; + sprintf(file1, "%s%sfile1", dirName, FastOS_File::GetPathSeparator()); + sprintf(file2, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); + sprintf(file3, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); + sprintf(subdir1, "%s%sdir1", dirName, FastOS_File::GetPathSeparator()); + sprintf(subdir2, "%s%sdir2", dirName, FastOS_File::GetPathSeparator()); + sprintf(file4, "%s%sfile4", subdir2, FastOS_File::GetPathSeparator()); + sprintf(file5, "%s%sfile5", subdir2, FastOS_File::GetPathSeparator()); + + FastOS_StatInfo statInfo; + + bool success = false; + + // Don't run at all if the directory already exists + assert(!FastOS_File::Stat(dirName, &statInfo)); + + FastOS_File::MakeDirectory(dirName); + + // Verify that we succeed with an empty directory + FastOS_File::EmptyDirectory(dirName); + success = FastOS_File::Stat(dirName, &statInfo); + Progress(success, "Removing empty directory."); + + // Verify that we can empty a directory with files in it + createFile(file1); + createFile(file2); + createFile(file3); + FastOS_File::EmptyDirectory(dirName); + success = + !FastOS_File::Stat(file1, &statInfo) && + !FastOS_File::Stat(file2, &statInfo) && + !FastOS_File::Stat(file3, &statInfo) && + FastOS_File::Stat(dirName, &statInfo); + Progress(success, "Deleting dir with files in it."); + + // Verify that we can empty a directory with files and directories in it + createFile(file1); + createFile(file2); + createFile(file3); + FastOS_File::MakeDirectory(subdir1); + FastOS_File::MakeDirectory(subdir2); + createFile(file4); + createFile(file5); + FastOS_File::EmptyDirectory(dirName); + success = FastOS_File::Stat(dirName, &statInfo) && + !FastOS_File::Stat(file1, &statInfo) && + !FastOS_File::Stat(file2, &statInfo) && + !FastOS_File::Stat(file3, &statInfo) && + !FastOS_File::Stat(file4, &statInfo) && + !FastOS_File::Stat(file5, &statInfo) && + !FastOS_File::Stat(subdir1, &statInfo) && + !FastOS_File::Stat(subdir2, &statInfo); + Progress(success, "Emptying directory with files and folders in it."); + + // Verify that we don't empty the directory if we find a file to keep + createFile(file1); + createFile(file2); + createFile(file3); + FastOS_File::MakeDirectory(subdir1); + FastOS_File::MakeDirectory(subdir2); + createFile(file4); + createFile(file5); + FastOS_File::EmptyDirectory(dirName, "file1"); + success = FastOS_File::Stat(dirName, &statInfo); + Progress(success, "Emptying dir with keepfile in it."); + // Verify that all but the file to keep are removed + success = FastOS_File::Stat(file1, &statInfo) && + !FastOS_File::Stat(file2, &statInfo) && + !FastOS_File::Stat(file3, &statInfo) && + !FastOS_File::Stat(file4, &statInfo) && + !FastOS_File::Stat(file5, &statInfo) && + !FastOS_File::Stat(subdir1, &statInfo) && + !FastOS_File::Stat(subdir2, &statInfo); + Progress(success, "Looking for keepfile."); + + // Verify that we don't empty the sub-directory if we find a file to keep + createFile(file1); + createFile(file2); + createFile(file3); + FastOS_File::MakeDirectory(subdir1); + FastOS_File::MakeDirectory(subdir2); + createFile(file4); + createFile(file5); + FastOS_File::EmptyDirectory(dirName, "file4"); + success = FastOS_File::Stat(dirName, &statInfo); + Progress(success, "Emptying file with nested keepfile."); + // Verify that all but the file to keep are removed + success = !FastOS_File::Stat(file1, &statInfo) && + !FastOS_File::Stat(file2, &statInfo) && + !FastOS_File::Stat(file3, &statInfo) && + FastOS_File::Stat(file4, &statInfo) && + !FastOS_File::Stat(file5, &statInfo) && + !FastOS_File::Stat(subdir1, &statInfo) && + FastOS_File::Stat(subdir2, &statInfo); + // Progress(success, "Looking for nested keepfile."); // Unsupported for now. + + + FastOS_File::EmptyAndRemoveDirectory(dirName); + + FastOS_File::MakeDirectory(dirName); + + // Verify that we can remove an empty directory + FastOS_File::EmptyAndRemoveDirectory(dirName); + success = !FastOS_File::Stat(dirName, &statInfo); + Progress(success, "Deleting empty directory."); + + // Verify that we can remove a directory with files in it + FastOS_File::MakeDirectory(dirName); + createFile(file1); + createFile(file2); + createFile(file3); + FastOS_File::EmptyAndRemoveDirectory(dirName); + success = !FastOS_File::Stat(dirName, &statInfo); + Progress(success, "Deleting a directory with files in it."); + + // Verify that we can remove a directory with files and directories in it + FastOS_File::MakeDirectory(dirName); + createFile(file1); + createFile(file2); + createFile(file3); + FastOS_File::MakeDirectory(subdir1); + FastOS_File::MakeDirectory(subdir2); + createFile(file4); + createFile(file5); + FastOS_File::EmptyAndRemoveDirectory(dirName); + success = !FastOS_File::Stat(dirName, &statInfo); + Progress(success, "Deleting directory with files and directories in it."); + + } + + void MoveFileTest() { + TestHeader ("Moving files (across volumes too) test"); + + const char *dirName = "tmpTestDir"; + char file1[1024]; + char file2[1024]; + char file3[1024]; + sprintf(file1, "%s%sfile1", dirName, FastOS_File::GetPathSeparator()); + sprintf(file2, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); + sprintf(file3, "%stmp%sfile3", FastOS_File::GetPathSeparator(), + FastOS_File::GetPathSeparator()); + + FastOS_File::MakeDirectory(dirName); + createFile(file1); + + FastOS_StatInfo statInfo; + // Move file to new name in same dir. + FastOS_File::MoveFile(file1, file2); + Progress(FastOS_File::Stat(file2, &statInfo), "Moving one within a directory."); + + // Move file to /tmp. + FastOS_File::MoveFile(file2, file3); + Progress(FastOS_File::Stat(file3, &statInfo), "Moving to /tmp/"); + + // Clean up + FastOS_File::Delete(file3); + FastOS_File::EmptyAndRemoveDirectory(dirName); + } + + void GetCurrentDirTest () + { + TestHeader ("Get Current Directory Test"); + + std::string currentDir = FastOS_File::getCurrentDirectory(); + + Progress(!currentDir.empty(), + "Current dir: %s", !currentDir.empty() ? + currentDir.c_str() : "<failed>"); + + bool dirrc = FastOS_File::SetCurrentDirectory(".."); + + std::string parentDir; + + if (dirrc) { + parentDir = FastOS_File::getCurrentDirectory(); + } + + Progress(dirrc && strcmp(currentDir.c_str(), parentDir.c_str()) != 0, + "Parent dir: %s", !parentDir.empty() ? + parentDir.c_str() : "<failed>"); + + dirrc = FastOS_File::SetCurrentDirectory(currentDir.c_str()); + + Progress(dirrc, "Changed back to working directory."); + + PrintSeparator(); + } + + void MemoryMapTest () + { + TestHeader ("Memory Map Test"); + + int i; + const int bufSize = 1000; + + FastOS_File file("generated/memorymaptest"); + + bool rc = file.OpenReadWrite(); + Progress(rc, "Opening file 'generated/memorymaptest'"); + + if(rc) + { + char *buffer = new char [bufSize]; + for (i = 0; i < bufSize; i++) + buffer[i] = i % 256; + + ssize_t wroteB = file.Write2(buffer, bufSize); + Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); + + file.Close(); + + file.enableMemoryMap(0); + + rc = file.OpenReadOnly(); + + Progress(rc, "Opening file 'generated/memorymaptest' read-only"); + if(rc) + { + bool mmapEnabled; + char *mmapBuffer = NULL; + + mmapEnabled = file.IsMemoryMapped(); + mmapBuffer = static_cast<char *>(file.MemoryMapPtr(0)); + + Progress(rc, "Memory mapping %s", + mmapEnabled ? "enabled" : "disabled"); + Progress(rc, "Map address: 0x%p", mmapBuffer); + + if (mmapEnabled) + { + rc = 0; + for (i = 0; i < bufSize; i++) + rc |= (mmapBuffer[i] == i % 256); + + Progress(rc, "Reading %d bytes from memory map", bufSize); + } + } + delete [] buffer; + } + PrintSeparator(); + } + + void DirectIOTest () + { + TestHeader ("Direct Disk IO Test"); + + int i; + const int bufSize = 40000; + + FastOS_File file("generated/diotest"); + + bool rc = file.OpenWriteOnly(); + Progress(rc, "Opening file 'generated/diotest' write-only"); + + if(rc) + { + char *buffer = new char [bufSize]; + + for(i=0; i<bufSize; i++) + buffer[i] = 'A' + (i % 17); + + ssize_t wroteB = file.Write2(buffer, bufSize); + Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); + + file.Close(); + + if(rc) + { + file.EnableDirectIO(); + + rc = file.OpenReadOnly(); + Progress(rc, "Opening file 'generated/diotest' read-only"); + if(rc) + { + bool dioEnabled; + size_t memoryAlignment=0; + size_t transferGranularity=0; + size_t transferMaximum=0; + + dioEnabled = file.GetDirectIORestrictions(memoryAlignment, + transferGranularity, + transferMaximum); + + Progress(rc, "DirectIO %s", dioEnabled ? "enabled" : "disabled"); + Progress(rc, "Memory alignment: %u bytes", memoryAlignment); + Progress(rc, "Transfer granularity: %u bytes", transferGranularity); + Progress(rc, "Transfer maximum: %u bytes", transferMaximum); + + if(dioEnabled) + { + int eachRead = (8192 + transferGranularity - 1) / transferGranularity; + + char *buffer2 = new char [(eachRead * transferGranularity + + memoryAlignment - 1)]; + char *alignPtr = buffer2; + unsigned int align = + static_cast<unsigned int> + (reinterpret_cast<unsigned long>(alignPtr) & + (memoryAlignment - 1)); + if(align != 0) + alignPtr = &alignPtr[memoryAlignment - align]; + + int residue = bufSize; + int pos=0; + while(residue > 0) + { + int readThisTime = eachRead * transferGranularity; + if(readThisTime > residue) + readThisTime = residue; + + file.ReadBuf(alignPtr, readThisTime, pos); + + for(i=0; i<readThisTime; i++) + { + rc = (alignPtr[i] == 'A' + ((i+pos) % 17)); + if(!rc) + { + Progress(false, "Read error at offset %d", i); + break; + } + } + residue -= readThisTime; + pos += readThisTime; + + if(!rc) + break; + } + if(rc) + { + Progress(true, "Read success"); + + rc = file.SetPosition(1); + Progress(rc, "SetPosition(1)"); + if(rc) + { + const int attemptReadBytes = 173; + ssize_t readB = file.Read(buffer, attemptReadBytes); + Progress(readB == attemptReadBytes, + "Read %d bytes successfully", + readB); + for(i = 0; i < attemptReadBytes; i++) + { + rc = (buffer[i] == 'A' + ((i+ 1) % 17)); + if(!rc) + { + Progress(false, "Read error at offset %d", i); + break; + } + } + } + if (rc) { + rc = file.SetPosition(1); + Progress(rc, "SetPosition(1)"); + if(rc) + { + const int attemptReadBytes = 4096; + ssize_t readB = file.Read(buffer, + attemptReadBytes); + Progress(readB == attemptReadBytes, + "Read %d bytes successfully", + readB); + for(i = 0; i < attemptReadBytes; i++) + { + rc = (buffer[i] == 'A' + ((i+ 1) % 17)); + if(!rc) + { + Progress(false, + "Read error at offset %d", i); + break; + } + } + } + } + } + delete [] buffer2; + } + else + { + memset(buffer, 0, bufSize); + + ssize_t readBytes = file.Read(buffer, bufSize); + Progress(readBytes == bufSize, + "Reading %d bytes from file", bufSize); + + for(i=0; i<bufSize; i++) + { + rc = (buffer[i] == 'A' + (i % 17)); + if(!rc) + { + Progress(false, "Read error at offset %d", i); + break; + } + } + if(rc) + Progress(true, "Read success"); + } + } + } + delete [] buffer; + } + + PrintSeparator(); + } + + void ReadOnlyTest () + { + TestHeader("Read-Only Test"); + + FastOS_File *myFile = new FastOS_File(roFilename); + + if(myFile->OpenReadOnly()) + { + int64_t filesize; + filesize = myFile->GetSize(); + + printf("%s: File size: %ld\n", + (filesize == 27) ? okString : failString, + static_cast<long>(filesize)); + + char dummyData[6] = "Dummy"; + bool writeResult = myFile->CheckedWrite(dummyData, 6); + + if(writeResult) + printf("FAILED: Should not be able to write a file opened for read-only access.\n"); + else + { + char dummyData2[28]; + printf("%s: Write failed with read-only access.\n", okString); + + bool rc = myFile->SetPosition(1); + Progress(rc, "Setting position to 1"); + + if(rc) + { + ssize_t readBytes; + int64_t filePosition; + readBytes = myFile->Read(dummyData2, 28); + + Progress(readBytes == 26, "Attempting to read 28 bytes, should get 26. Got: %d", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); + + readBytes = myFile->Read(dummyData2, 6); + Progress(readBytes == 0, "We should now get 0 bytes. Read: %d bytes", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); + } + } + } + else + printf("%s: Unable to open file 'hello.txt'.\n", failString); + + delete(myFile); + PrintSeparator(); + } + + void WriteOnlyTest () + { + TestHeader("Write-Only Test"); + + FastOS_File *myFile = new FastOS_File(woFilename); + + if(myFile->OpenWriteOnly()) + { + + int64_t filesize; + filesize = myFile->GetSize(); + + printf("%s: File size: %ld\n", + (filesize == 0) ? okString : failString, + static_cast<long>(filesize)); + + char dummyData[6] = "Dummy"; + bool writeResult = myFile->CheckedWrite(dummyData, 6); + + if(!writeResult) + { + printf("%s: Should be able to write to file opened for write-only access.\n", failString); + } + else + { + printf("%s: Write 6 bytes ok.\n", okString); + + int64_t filePosition = myFile->GetPosition(); + if(filePosition == 6) + { + printf("%s: Fileposition is now 6.\n", okString); + + if(myFile->SetPosition(0)) + { + printf("%s: SetPosition(0) success.\n", okString); + filePosition = myFile->GetPosition(); + + if(filePosition == 0) + { + printf("%s: Fileposition is now 0.\n", okString); + + int readBytes = myFile->Read(dummyData, 6); + + if(readBytes != 6) + { + printf("%s: Trying to read a write-only file should fail and it did.\n", okString); + printf("%s: Return code was: %d.\n", okString, readBytes); + } + else + printf("%s: Read on a file with write-only access should fail, but it didn't.\n", failString); + } + else + printf("%s: Fileposition should be 6, but was %ld.\n", + failString, + static_cast<long>(filePosition)); + } + else + printf("%s: SetPosition(0) failed\n", failString); + } + else + printf("%s: Fileposition should be 6, but was %ld.\n", + failString, + static_cast<long>(filePosition)); + } + + bool closeResult = myFile->Close(); + printf("%s: Close file.\n", closeResult ? okString : failString); + } + else + { + printf("%s: Unable to open file 'hello.txt'.\n", failString); + } + + + bool deleteResult = myFile->Delete(); + + printf("%s: Delete file '%s'.\n", deleteResult ? okString : failString, woFilename); + + + delete(myFile); + PrintSeparator(); + } + + void ReadWriteTest () + { + TestHeader("Read/Write Test"); + + FastOS_File *myFile = new FastOS_File(rwFilename); + + if(myFile->OpenExisting()) + { + printf("%s: OpenExisting() should not work when '%s' does not exist.\n", failString, rwFilename); + + myFile->Close(); + } + else + printf("%s: OpenExisting() should fail when '%s' does not exist, and it did.\n", okString, rwFilename); + + + if(myFile->OpenReadWrite()) + { + int64_t filesize; + + filesize = myFile->GetSize(); + + printf("%s: File size: %ld\n", + (filesize == 0) ? okString : failString, + static_cast<long>(filesize)); + + char dummyData[6] = "Dummy"; + + bool writeResult = myFile->CheckedWrite(dummyData, 6); + + if(!writeResult) + printf("%s: Should be able to write to file opened for read/write access.\n", failString); + else + { + printf("%s: Write 6 bytes ok.\n", okString); + + int64_t filePosition = myFile->GetPosition(); + + if(filePosition == 6) + { + printf("%s: Fileposition is now 6.\n", okString); + + if(myFile->SetPosition(0)) + { + printf("%s: SetPosition(0) success.\n", okString); + filePosition = myFile->GetPosition(); + + if(filePosition == 0) + { + printf("%s: Fileposition is now 0.\n", okString); + + char dummyData2[7]; + int readBytes = myFile->Read(dummyData2, 6); + + if(readBytes == 6) + { + printf("%s: Reading 6 bytes worked.\n", okString); + + int cmpResult = memcmp(dummyData, dummyData2, 6); + + printf("%s: Comparing the written and read result.\n", + (cmpResult == 0) ? okString : failString); + + bool rc = myFile->SetPosition(1); + Progress(rc, "Setting position to 1"); + + if(rc) + { + readBytes = myFile->Read(dummyData2, 7); + + Progress(readBytes == 5, "Attempting to read 7 bytes, should get 5. Got: %d", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); + + readBytes = myFile->Read(dummyData2, 6); + Progress(readBytes == 0, "We should not be able to read any more. Read: %d bytes", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); + } + } + else + printf("%s: Reading 6 bytes failed.\n", failString); + } + else + printf("%s: Fileposition should be 6, but was %ld.\n", + failString, + static_cast<long>(filePosition)); + } + else + printf("%s: SetPosition(0) failed\n", failString); + } + else + printf("%s: Fileposition should be 6, but was %ld.\n", + failString, + static_cast<long>(filePosition)); + } + + bool closeResult = myFile->Close(); + + printf("%s: Close file.\n", closeResult ? okString : failString); + } + else + printf("%s: Unable to open file 'hello.txt'.\n", failString); + + bool deleteResult = myFile->Delete(); + printf("%s: Delete file '%s'.\n", deleteResult ? okString : failString, rwFilename); + + delete(myFile); + PrintSeparator(); + } + + void ScanDirectoryTest() + { + TestHeader("DirectoryScan Test"); + + FastOS_DirectoryScan *scanDir = new FastOS_DirectoryScan("."); + + while(scanDir->ReadNext()) + { + const char *name = scanDir->GetName(); + bool isDirectory = scanDir->IsDirectory(); + bool isRegular = scanDir->IsRegular(); + + printf("%-30s %s\n", name, + isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); + } + + delete(scanDir); + PrintSeparator(); + } + + void ReadBufTest () + { + TestHeader("ReadBuf Test"); + + FastOS_File file("hello.txt"); + + char buffer[20]; + + if(file.OpenReadOnly()) + { + int64_t position = file.GetPosition(); + Progress(position == 0, "File pointer should be 0 after opening file"); + + file.Read(buffer, 4); + buffer[4] = '\0'; + position = file.GetPosition(); + Progress(position == 4, "File pointer should be 4 after reading 4 bytes"); + Progress(strcmp(buffer, "This") == 0, "[This]=[%s]", buffer); + + file.ReadBuf(buffer, 6, 8); + buffer[6] = '\0'; + position = file.GetPosition(); + Progress(position == 4, "File pointer should still be 4 after ReadBuf"); + Progress(strcmp(buffer, "a test") == 0, "[a test]=[%s]", buffer); + + file.Close(); + } + + PrintSeparator(); + } + + void DiskFreeSpaceTest () + { + TestHeader("DiskFreeSpace Test"); + + int64_t freeSpace = FastOS_File::GetFreeDiskSpace("hello.txt"); + ProgressI64(freeSpace != -1, "DiskFreeSpace using file (hello.txt): %" + PRId64 " MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); + freeSpace = FastOS_File::GetFreeDiskSpace("."); + ProgressI64(freeSpace != -1, "DiskFreeSpace using dir (.): %" + PRId64 " MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); + PrintSeparator(); + } + + void MaxLengthTest () + { + TestHeader ("Max Lengths Test"); + + int maxval = FastOS_File::GetMaximumFilenameLength("."); + Progress(maxval > 5 && maxval < (512*1024), + "Maximum filename length = %d", maxval); + + maxval = FastOS_File::GetMaximumPathLength("."); + Progress(maxval > 5 && maxval < (512*1024), + "Maximum path length = %d", maxval); + + PrintSeparator(); + } + + void CopyFileTest () + { + FastOS_StatInfo statInfo; + TestHeader("CopyFile Test"); + const char *dirName = "tmpDir"; + char file1[1024]; + char file2[1024]; + char file3[1024]; + char file4[1024]; + char file5[1024]; + sprintf(file1, "%s%sfile1", dirName, FastOS_File::GetPathSeparator()); + sprintf(file2, "%s%sfile2", dirName, FastOS_File::GetPathSeparator()); + sprintf(file3, "%s%sfile3", dirName, FastOS_File::GetPathSeparator()); + sprintf(file4, "%s%sfile4", dirName, FastOS_File::GetPathSeparator()); + sprintf(file5, "%s%sfile5", dirName, FastOS_File::GetPathSeparator()); + + FastOS_File::EmptyAndRemoveDirectory(dirName); + FastOS_File::MakeDirectory(dirName); + printf("Creating files to copy. Some of them are quite large...\n\n"); + createFile(file1); + createFile(file3, 20*1024*1024); // 20MB file. + createFile(file4, 1024*1024); // 1MB file, i.e. exact size of buffer. + createFile(file5, 1024*1024 + 100); // 1.001MB file + + FastOS_File::Stat(file4, &statInfo); + unsigned int sizeOfFile4 = statInfo._size; + + FastOS_File::Stat(file5, &statInfo); + unsigned int sizeOfFile5 = statInfo._size; + + // Tests start here. + bool copyOK = FastOS_File::CopyFile(file1, file2); + Progress(copyOK, + "File copy from %s to %s.", file1, file2); + + FastOS_File::Delete(file2); + copyOK = FastOS_File::CopyFile(file3, file2); + Progress(copyOK, + "File copy from %s to %s.", file3, file2); + FastOS_File::Stat(file2, &statInfo); + Progress(statInfo._size == 20*1024*1024, + "Size of copied file is 20MB."); + + copyOK = FastOS_File::CopyFile(file3, file3); + Progress(!copyOK, + "File copy onto itself should fail."); + + FastOS_File::Delete(file1); + copyOK = FastOS_File::CopyFile(file1, file2); + Progress(!copyOK, + "File copy of a missing file should fail."); + + copyOK = FastOS_File::CopyFile(file4, file2); + Progress(copyOK, + "Copying a smaller file onto a larger one."); + FastOS_File::Stat(file2, &statInfo); + Progress(statInfo._size == sizeOfFile4, + "Size of copied file should be %u bytes.", sizeOfFile4); + + copyOK = FastOS_File::CopyFile(file4, file1); + Progress(copyOK, + "Copying a file with exact size of buffer."); + FastOS_File::Stat(file1, &statInfo); + Progress(statInfo._size == sizeOfFile4, + "Size of copied file should be %u bytes.", sizeOfFile4); + + copyOK = FastOS_File::CopyFile(file5, file1); + Progress(copyOK, + "Copying a file with size %u bytes.", sizeOfFile5); + FastOS_File::Stat(file1, &statInfo); + Progress(statInfo._size == sizeOfFile5, + "Size of copied file should be %u bytes.", sizeOfFile5); + + + FastOS_File::EmptyAndRemoveDirectory("./tmpDir"); + PrintSeparator(); + } + + int Main () + { + printf("This test should be run in the 'test/workarea' directory.\n\n"); + printf("grep for the string '%s' to detect failures.\n\n", failString); + + DirectoryTest(); + MoveFileTest(); + CopyFileTest(); + GetCurrentDirTest(); + DirectIOTest(); + MaxLengthTest(); + DiskFreeSpaceTest(); + ReadOnlyTest(); + WriteOnlyTest(); + ReadWriteTest(); + ScanDirectoryTest(); + ReadBufTest(); + MemoryMapTest(); + + PrintSeparator(); + printf("END OF TEST (%s)\n", _argv[0]); + + return 0; + } +}; + +const char *FileTest::roFilename = "hello.txt"; +const char *FileTest::woFilename = "generated/writeonlytest.txt"; +const char *FileTest::rwFilename = "generated/readwritetest.txt"; + + +int main (int argc, char **argv) +{ + FileTest app; + + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} diff --git a/fastos/src/tests/mazeserver.cpp b/fastos/src/tests/mazeserver.cpp new file mode 100644 index 00000000000..bdd6ae50db0 --- /dev/null +++ b/fastos/src/tests/mazeserver.cpp @@ -0,0 +1,4 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#define DO_MAZE_SERVER 1 + +#include "sockettest.cpp" diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp new file mode 100644 index 00000000000..3865920f981 --- /dev/null +++ b/fastos/src/tests/performancetest.cpp @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> + +#include <vespa/fastos/fastos.h> +#include "tests.h" + +void PerformanceTest (char *buffer); + +int main (int argc, char **argv) +{ + (void)argc; + (void)argv; + + void (*test)(char *buffer) = PerformanceTest; + + test(NULL); + return 0; +} + +void PerformanceTest (char *buffer) +{ + // Cause exception + *static_cast<char *>(NULL) = 'e'; + +#if 1 + FastOS_File file("test.txt"); + + if(file.OpenReadOnly()) + { + file.Read(buffer, 20); + file.Write2(buffer, 20); + file.Read(buffer, 20); + file.Write2(buffer, 20); + file.Read(buffer, 20); + file.Write2(buffer, 20); + } +#else + + int filedes = open("test.txt", O_RDONLY, 0664); + + if(filedes != -1) + { + write(filedes, buffer, 20); + read(filedes, buffer, 20); + write(filedes, buffer, 20); + read(filedes, buffer, 20); + write(filedes, buffer, 20); + read(filedes, buffer, 20); + write(filedes, buffer, 20); + + close(filedes); + } +#endif +} + diff --git a/fastos/src/tests/prefetchtest.cpp b/fastos/src/tests/prefetchtest.cpp new file mode 100644 index 00000000000..efb68be0362 --- /dev/null +++ b/fastos/src/tests/prefetchtest.cpp @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * FastOS_Prefetch test program. + * + * @author Olaf Birkeland + * @version $Id$ + */ + /* + * Creation date : 2000-12-11 + * Copyright (c) : 1997-2001 Fast Search & Transfer ASA + * ALL RIGHTS RESERVED + *************************************************************************/ + + + +#include <stdlib.h> +#include <math.h> + +#include <vespa/fastos/fastos.h> +#include "tests.h" + + +class PrefetchTestApp : public BaseTest +{ +private: + +public: + virtual ~PrefetchTestApp() {}; + + bool PrefetchTest () + { + bool rc = false; + int j, size, *a; + register int or1, or2; + FastOS_Time start, stop; + double timeVal; + + TestHeader("Prefetch Test"); + + // 32MB + size = 32; + size *= 1024*1024/sizeof(*a); + + if ((a = static_cast<int *>(calloc(size, sizeof(*a)))) != NULL) + { + // Standard loop + start.SetNow(); + or1 = 1; + for(j=0; j<size; j++) + or1 |= a[j]; + stop.SetNow(); + timeVal = stop.MilliSecs() - start.MilliSecs(); + Progress(or1==1, "Result = %d", or1); + ProgressFloat(true, "%4.3f MB/s (standard loop)", + float(size*sizeof(*a)/(1E3*timeVal))); + + + // Unrolled loop + start.SetNow(); + or1 = or2 = 2; + for(j=0; j<size; j+=8) + { + or1 |= a[j+0]|a[j+1]|a[j+2]|a[j+3]; + or2 |= a[j+4]|a[j+5]|a[j+6]|a[j+7]; + } + or1 |= or2; + stop.SetNow(); + timeVal = stop.MilliSecs() - start.MilliSecs(); + Progress(or1 == 2, "Result = %d", or1); + ProgressFloat(true, "%4.3f MB/s (unrolled loop)", + float(size*sizeof(*a)/(1E3*timeVal))); + + + // Unrolled loop with prefetch + start.SetNow(); + or1 = or2 = 3; + for(j=0; j<size; j+=8) + { + FastOS_Prefetch::NT(&a[j+32]); + or1 |= a[j+0]|a[j+1]|a[j+2]|a[j+3]; + or2 |= a[j+4]|a[j+5]|a[j+6]|a[j+7]; + } + or1 |= or2; + stop.SetNow(); + timeVal = stop.MilliSecs() - start.MilliSecs(); + Progress(or1 == 3, "Result = %d", or1); + ProgressFloat(true, "%4.3f MB/s (unrolled loop with prefetch)", + float(size*sizeof(*a)/(1E3*timeVal))); + + // Unrolled loop + start.SetNow(); + or1 = or2 = 4; + for(j=0; j<size; j+=8) + { + or1 |= a[j+0]|a[j+1]|a[j+2]|a[j+3]; + or2 |= a[j+4]|a[j+5]|a[j+6]|a[j+7]; + } + or1 |= or2; + stop.SetNow(); + timeVal = stop.MilliSecs() - start.MilliSecs(); + Progress(or1 == 4, "Result = %d", or1); + ProgressFloat(true, "%4.3f MB/s (unrolled loop)", + float(size*sizeof(*a)/(1E3*timeVal))); + + + // Standard loop + start.SetNow(); + or1 = 5; + for(j=0; j<size; j++) + or1 |= a[j]; + stop.SetNow(); + timeVal = stop.MilliSecs() - start.MilliSecs(); + Progress(or1 == 5, "Result = %d", or1); + ProgressFloat(true, "%4.3f MB/s (standard loop)", + float(size*sizeof(*a)/(1E3*timeVal))); + + + // Unrolled loop with prefetch + start.SetNow(); + or1 = or2 = 6; + for(j=0; j<size; j+=8) + { + FastOS_Prefetch::NT(&a[j+32]); + or1 |= a[j+0]|a[j+1]|a[j+2]|a[j+3]; + or2 |= a[j+4]|a[j+5]|a[j+6]|a[j+7]; + } + or1 |= or2; + stop.SetNow(); + timeVal = stop.MilliSecs() - start.MilliSecs(); + Progress(or1 == 6, "Result = %d", or1); + ProgressFloat(true, "%4.3f MB/s (unrolled loop with prefetch)", + float(size*sizeof(*a)/(1E3*timeVal))); + + + free(a); + rc = true; + } + else + Progress(false, "Out of memory!!"); + + PrintSeparator(); + + return rc; + } + + int Main () + { + int rc = 1; + printf("grep for the string '%s' to detect failures.\n\n", failString); + + if(PrefetchTest()) + rc = 0; + + printf("END OF TEST (%s)\n", _argv[0]); + + return rc; + } +}; + + +int main (int argc, char **argv) +{ + PrefetchTestApp app; + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} diff --git a/fastos/src/tests/processtest.cpp b/fastos/src/tests/processtest.cpp new file mode 100644 index 00000000000..30b4570a776 --- /dev/null +++ b/fastos/src/tests/processtest.cpp @@ -0,0 +1,502 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "tests.h" + +class MyListener : public FastOS_ProcessRedirectListener +{ +private: + MyListener(const MyListener&); + MyListener& operator=(const MyListener&); + + const char *_title; + int _receivedBytes; + +public: + static int _allocCount; + static int _successCount; + static int _failCount; + static FastOS_Mutex *_counterLock; + + MyListener (const char *title) + : _title(title), + _receivedBytes(0) + { + _counterLock->Lock(); + _allocCount++; + _counterLock->Unlock(); + } + + virtual ~MyListener () + { + bool isStdout = (strcmp(_title, "STDOUT") == 0); + + const int correctByteCount = 16; + + _counterLock->Lock(); + if(_receivedBytes == (isStdout ? correctByteCount : 0)) + _successCount++; + else + _failCount++; + + _allocCount--; + _counterLock->Unlock(); + } + + void OnReceiveData (const void *data, size_t length) + { + _receivedBytes += length; + if(data != NULL) + { +#if 0 + printf("[%s] received %u bytes of data:\n%s\n", + _title, length, static_cast<const char *>(data)); +#endif + } + else + delete(this); + } +}; + +int MyListener::_allocCount = 0; +int MyListener::_successCount = 0; +int MyListener::_failCount = 0; +FastOS_Mutex *MyListener::_counterLock = NULL; + + +class ThreadRunJob : public FastOS_Runnable +{ +private: + ThreadRunJob(const ThreadRunJob&); + ThreadRunJob& operator=(const ThreadRunJob&); + + FastOS_Process::Priority _processPriority; + const char *_processCmdLine; + int _timeSpent; +public: + ThreadRunJob (FastOS_Process::Priority processPriority, + const char *commandLine) : + _processPriority(processPriority), + _processCmdLine(commandLine), + _timeSpent(0) + { + } + + void Run (FastOS_ThreadInterface *thisThread, void *arg) + { + (void)thisThread; + (void)arg; + + FastOS_Time startTime, endTime; + startTime.SetNow(); + + FastOS_Process xproc(_processCmdLine); + int returnCode = -1; + + if(xproc.Create()) + { + xproc.SetPriority(_processPriority); + xproc.Wait(&returnCode); + } + + endTime.SetNow(); + + endTime -= startTime; + + _timeSpent = int(endTime.MilliSecs()); + } + + int GetTimeSpent () + { + return _timeSpent; + } +}; + +class ProcessTest : public BaseTest +{ +private: + virtual bool useProcessStarter() const { return true; } + virtual bool useIPCHelper() const { return true; } + ProcessTest(const ProcessTest&); + ProcessTest& operator=(const ProcessTest&); + + int GetLastError () + { + return errno; + } + + // Flag which indicates whether an IPC message is received + // or not. + bool _gotMessage; + int _receivedMessages; + FastOS_Mutex *_counterLock; + bool _isChild; +public: + ProcessTest () + : _gotMessage(false), + _receivedMessages(0), + _counterLock(NULL), + _isChild(true) + { + } + + void OnReceivedIPCMessage (const void *data, size_t length) + { + // printf("Data: [%s]\n", static_cast<const char *>(data)); + + if(length == 5) + { + const char *dataMatch = "IPCM"; + if(!_isChild) + dataMatch = "IPCR"; + if(strcmp(static_cast<const char *>(data), dataMatch) != 0) + Progress(false, + "Received message did not match \"%s\" (%s)", + dataMatch, static_cast<const char *>(data)); + } + else + Progress(false, + "Received message was not 5 bytes long (%d)", length); + + _gotMessage = true; + + // We only have the counter lock if we are the parent process. + if(_counterLock != NULL) + { + _counterLock->Lock(); + _receivedMessages++; + _counterLock->Unlock(); + } + } + + void PollWaitTest () + { + TestHeader("PollWait Test"); + + FastOS_Process *xproc = new FastOS_Process("sort", true); + + if(xproc->Create()) + { + int i; + for(i=0; i<10; i++) + { + bool stillRunning; + int returnCode; + + if(!xproc->PollWait(&returnCode, &stillRunning)) + { + Progress(false, "PollWait failure: %d", + GetLastError()); + break; + } + + if(i <= 5) + Progress(stillRunning, "StillRunning = %s", + stillRunning ? "true" : "false"); + + if(!stillRunning) + { + Progress(returnCode == 0, "Process exit code: %d", + returnCode); + break; + } + + if(i == 5) + { + // Make sort quit + xproc->WriteStdin(NULL, 0); + } + + FastOS_Thread::Sleep(1000); + } + + if(i == 10) + { + Progress(false, "Timeout"); + xproc->Kill(); + } + } + delete xproc; + + PrintSeparator(); + } + + void ProcessTests (bool doKill, bool stdinPre, bool waitKill) + { + const int numLoops = 100; + const int numEachTime = 40; + + MyListener::_counterLock = new FastOS_Mutex(); + + char testHeader[200]; + strcpy(testHeader, "Process Test"); + if(doKill) + strcat(testHeader, " w/Kill"); + if(!stdinPre) + strcat(testHeader, " w/open stdin"); + if(waitKill) + strcat(testHeader, " w/Wait timeout"); + + TestHeader(testHeader); + + MyListener::_allocCount = 0; + MyListener::_successCount = 0; + MyListener::_failCount = 0; + + Progress(true, "Starting processes..."); + + for(int i=0; i<numLoops; i++) + { + FastOS_ProcessInterface *procs[numEachTime]; + + int j; + for(j=0; j<numEachTime; j++) + { + FastOS_ProcessInterface *xproc = + new FastOS_Process("sort", true, + new MyListener("STDOUT"), + new MyListener("STDERR")); + + if(xproc->Create()) + { + const char *str = "Peter\nPaul\nMary\n"; + + if(!waitKill && stdinPre) + { + xproc->WriteStdin(str, strlen(str)); + xproc->WriteStdin(NULL, 0); + } + + if(doKill) + { + if(!xproc->Kill()) + Progress(false, "Kill failure %d", GetLastError()); + } + + if(!waitKill && !stdinPre) + { + xproc->WriteStdin(str, strlen(str)); + xproc->WriteStdin(NULL, 0); + } + } + else + { + Progress(false, "Process.CreateWithShell failure %d", + GetLastError()); + delete xproc; + xproc = NULL; + } + procs[j] = xproc; + } + + for(j=0; j<numEachTime; j++) + { + FastOS_ProcessInterface *xproc = procs[j]; + if(xproc == NULL) + continue; + + int timeOut = -1; + if(waitKill) + timeOut = 1; + + FastOS_Time timeBeforeWait; + timeBeforeWait.SetNow(); + + int returnCode; + if(!xproc->Wait(&returnCode, timeOut)) + Progress(false, "Process.Wait failure %d", GetLastError()); + else + { + int checkReturnCode = 0; + if(doKill || waitKill) + checkReturnCode = FastOS_Process::KILL_EXITCODE; + if(returnCode != checkReturnCode) + Progress(false, "returnCode = %d", returnCode); + } + + if(waitKill) + { + FastOS_Time waitTime; + waitTime.SetNow(); + + waitTime -= timeBeforeWait; + double milliSecs = waitTime.MilliSecs(); + if((milliSecs < 900) || + (milliSecs > 3500)) + { + Progress(false, "WaitKill time = %d", int(milliSecs)); + } + } + + delete xproc; + + if(waitKill) + Progress(true, "Started %d processes", i * numEachTime + j + 1); + } + + if(!waitKill && ((i % 10) == 9)) + Progress(true, "Started %d processes", (i+1) * numEachTime); + + if(waitKill && (((i+1) * numEachTime) > 50)) + break; + } + + Progress(MyListener::_allocCount == 0, "MyListener alloc count = %d", + MyListener::_allocCount); + + if(!doKill && !waitKill) + { + Progress(MyListener::_successCount == (2 * numLoops * numEachTime), + "MyListener _successCount = %d", MyListener::_successCount); + + Progress(MyListener::_failCount == 0, + "MyListener _failCount = %d", MyListener::_failCount); + } + + delete MyListener::_counterLock; + MyListener::_counterLock = NULL; + + PrintSeparator(); + } + + int DoChildRole () + { + int rc = 124; + int i; + for(i=0; i<(20*10); i++) + { + if(_gotMessage) + break; + + FastOS_Thread::Sleep(100); + } + if(i < (20*10)) + { + // Send a message to the parent process. + const char *messageString = "IPCR"; + SendParentIPCMessage(messageString, strlen(messageString) + 1); + rc = 123; + } + else + Progress(false, "Child timed out waiting for IPC message"); + + return rc; + } + + void IPCTest () + { + TestHeader ("IPC Test"); + const char *childProgram = _argv[1]; + + _counterLock = new FastOS_Mutex(); + + int i; + for(i=0; i<30; i++) + { + FastOS_Process process(childProgram); + if(process.Create()) + { + // Send a message to the child process. + const char *messageString = "IPCM"; + process.SendIPCMessage(messageString, + strlen(messageString) + 1); + + // Wait for the process to end. + int returnCode; + if(process.Wait(&returnCode)) + { + Progress(returnCode == 123, + "Child exited with code: %d", returnCode); + } + else + Progress(false, "process.Wait() failed"); + } + else + Progress(false, "process.Create() (%s) failed", childProgram); + } + + Progress(_receivedMessages == i, + "Received %d messages", _receivedMessages); + + delete _counterLock; + _counterLock = NULL; + + PrintSeparator(); + } + + void PriorityTest () + { + TestHeader("Process Priority Test"); + + ThreadRunJob job1(FastOS_Process::PRIORITY_LOWEST, _argv[2]); + ThreadRunJob job2(FastOS_Process::PRIORITY_HIGHEST, _argv[2]); + FastOS_ThreadPool pool(128*1024); + Progress(true, "Starting usecpu-processes with low and high priority..."); + pool.NewThread(&job1); + pool.NewThread(&job2); + Progress(true, "Waiting for proccesses to finish..."); + pool.Close(); + int timeJob1 = job1.GetTimeSpent(); + int timeJob2 = job2.GetTimeSpent(); + + Progress(timeJob1 > timeJob2, "Job1 (low priority) %d ms", timeJob1); + Progress(timeJob1 > timeJob2, "Job2 (high priority) %d ms", timeJob2); + + PrintSeparator(); + } + + void NoInheritTest () + { + TestHeader("No Inherit Test"); + + const char *filename = "process__test.tmp___"; + + Progress(true, "Opening '%s' for writing...", filename); + void *inheritData = FastOS_Process::PrefopenNoInherit(); + FILE *fp = fopen(filename, "w"); + int numProc = FastOS_Process::PostfopenNoInherit(inheritData); + Progress(fp != NULL, "Open file"); + if(fp != NULL) + { + Progress(numProc > 0, "Number of files processed = %d\n", + numProc); + fclose(fp); + } + FastOS_File::Delete(filename); + + PrintSeparator(); + } + + int Main () + { + // This process is started as either a parent or a child. + // When the parent role is desired, a child program is supplied + // as argv[1] + + if(_argc != 3) + return DoChildRole(); + + _isChild = false; + + printf("grep for the string '%s' to detect failures.\n\n", failString); + + NoInheritTest(); + PriorityTest(); + PollWaitTest(); + IPCTest(); + ProcessTests(false, true, false); + ProcessTests(true, true, false); + ProcessTests(true, false, false); + ProcessTests(false, true, true); + + printf("END OF TEST (%s)\n", _argv[0]); + + return 0; + } +}; + +int main (int argc, char **argv) +{ + ProcessTest app; + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} diff --git a/fastos/src/tests/sockettest.cpp b/fastos/src/tests/sockettest.cpp new file mode 100644 index 00000000000..f0be5c0fddb --- /dev/null +++ b/fastos/src/tests/sockettest.cpp @@ -0,0 +1,925 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <assert.h> +#include <string.h> +#include <sstream> + +#include "tests.h" + + + +#define MAZE_FILE_OFFSET 1078 + +#define MAZE_WIDTH 776 +#define MAZE_HEIGHT 483 +#define MAZE_START_X 3 +#define MAZE_START_Y 399 +#define MAZE_END_X 759 +#define MAZE_END_Y 63 + +#define MAZE_WALL 0 +#define MAZE_EXIT_LEFT 1 +#define MAZE_EXIT_RIGHT 2 + +#define MAZE_DIRECTION_EAST 0 +#define MAZE_DIRECTION_SOUTH 1 +#define MAZE_DIRECTION_WEST 2 +#define MAZE_DIRECTION_NORTH 3 + +#define MAZE_QUERY_HALLWAY 1234 +#define MAZE_QUERY_QUIT 1235 + + +#define MAX_CONNECTIONS 20 + +#define SEARCHBUF_SIZE (1024*1024/(sizeof(u_short))) + + +class MazeServices +{ +public: + int TurnLeft (int direction) + { + direction--; + if(direction == -1) + direction = 3; + return direction; + } + + int TurnRight (int direction) + { + return (direction+1) % 4; + } + + void SetDirection(int &dx, int &dy, int direction) + { + switch(direction) + { + case MAZE_DIRECTION_EAST: + dx = 2; + dy = 0; + break; + + case MAZE_DIRECTION_SOUTH: + dx = 0; + dy = 2; + break; + + case MAZE_DIRECTION_WEST: + dx = -2; + dy = 0; + break; + + case MAZE_DIRECTION_NORTH: + dx = 0; + dy = -2; + break; + } + } + virtual ~MazeServices(void) { } +}; + +class MazeServer; + + +#define BUFFER_SIZE 20000 +class MazeServerConnection +{ +private: + MazeServerConnection(const MazeServerConnection&); + MazeServerConnection& operator=(const MazeServerConnection&); + + MazeServer *_server; + u_short _receiveBuffer[BUFFER_SIZE/sizeof(u_short)]; + u_short _sendBuffer[BUFFER_SIZE/sizeof(u_short)]; + unsigned char *_sendPtr, *_receivePtr; + int _bytesToSend; + + int ReceiveBufferSpace () + { + return static_cast<int> + (reinterpret_cast<unsigned char *>(&_receiveBuffer[BUFFER_SIZE]) - + _receivePtr); + } + + int SendBufferSpace () + { + return static_cast<int> + (reinterpret_cast<unsigned char *>(&_sendBuffer[BUFFER_SIZE]) - + _sendPtr); + } + + unsigned int ReceiveBufferBytes () + { + return BUFFER_SIZE - ReceiveBufferSpace(); + } + + unsigned int SendBufferBytes () + { + return BUFFER_SIZE - SendBufferSpace(); + } + +public: + FastOS_Socket *_socket; + bool _shouldFree; + + MazeServerConnection (MazeServer *server, FastOS_Socket *sock) + : _server(server), + _sendPtr(reinterpret_cast<unsigned char *>(_sendBuffer)), + _receivePtr(reinterpret_cast<unsigned char *>(_receiveBuffer)), + _bytesToSend(0), + _socket(sock), + _shouldFree(false) + { + } + + bool ReadEvent (); + bool WriteEvent (); +}; + + +class MazeServer : public MazeServices +{ +private: + MazeServer(const MazeServer&); + MazeServer& operator=(const MazeServer&); + +public: + unsigned char _mazeBitmap[MAZE_HEIGHT][MAZE_WIDTH]; + unsigned char _mazeBitmap2[MAZE_HEIGHT][MAZE_WIDTH]; + + FastOS_ServerSocket *_serverSocket; + BaseTest *_app; + + MazeServer(BaseTest *app) + : _serverSocket(NULL), + _app(app) + { + } + + bool Initialize() + { + bool rc; + const char *filename = "mazebitmap.bmp"; + + FastOS_File file; + + rc = file.OpenReadOnly(filename); + + _app->Progress(rc, "Opening maze bitmap (%s)", filename); + + if(rc) + { + rc = file.SetPosition(MAZE_FILE_OFFSET); + + _app->Progress(rc, "Setting file position (%d)", MAZE_FILE_OFFSET); + + if(rc) + { + int readBytes = file.Read(_mazeBitmap, MAZE_WIDTH * MAZE_HEIGHT); + + rc = (readBytes == MAZE_WIDTH*MAZE_HEIGHT); + + _app->Progress(rc, "Reading %d bytes from '%s'", + MAZE_WIDTH*MAZE_HEIGHT, filename); + + if(rc) + { + int serverPort = 18334; + _serverSocket = new FastOS_ServerSocket(serverPort); + _app->Progress(_serverSocket != NULL, + "Creating ServerSocket instance"); + + _app->Progress(_serverSocket->SetSoBlocking(false), + "Set non-blocking"); + + _app->Progress(_serverSocket->Listen(), + "Bind socket to port %d. Listen for " + "incoming connections.", serverPort); + } + } + } + + return rc; + } + + void Run () + { + if(Initialize()) + { + MazeServerConnection *connections[MAX_CONNECTIONS], *conn; + FastOS_SocketEvent socketEvent; + int i; + + memset(connections, 0, sizeof(connections)); + + _serverSocket->SetSocketEvent(&socketEvent); + _serverSocket->EnableReadEvent(true); + + for(;;) + { + bool waitError=false; + + if(socketEvent.Wait(waitError, 200)) + { + if(socketEvent.QueryReadEvent(_serverSocket)) + { + FastOS_Socket *connSocket = _serverSocket->AcceptPlain(); + + _app->Progress(connSocket != NULL, "Accepting socket (%p)", + connSocket); + + for(i=0; i<MAX_CONNECTIONS; i++) + { + if(connections[i] == NULL) + { + // Found a free positions for the new connection + break; + } + } + if(i < MAX_CONNECTIONS) + { + if(connSocket != NULL) + { + connections[i] = new MazeServerConnection(this, connSocket); + + assert(connections[i] != NULL); + + connSocket->SetSocketEvent(&socketEvent); + connSocket->EnableReadEvent(true); + } + } + else + { + _app->Progress(false, "Rejecting connection. Only %d allowed.", MAX_CONNECTIONS); + delete(connSocket); + } + } + + for(i=0; i<MAX_CONNECTIONS; i++) + { + if((conn = connections[i]) != NULL) + { + if(socketEvent.QueryReadEvent(conn->_socket)) + { + if(conn->ReadEvent()) + conn->_socket->EnableWriteEvent(true); + } + + if(socketEvent.QueryWriteEvent(conn->_socket)) + { + if(!conn->WriteEvent()) + conn->_socket->EnableWriteEvent(false); + } + + if(conn->_shouldFree) + { + delete(conn); + connections[i] = NULL; + } + } + } + } + } + } + } + + int Read (int x, int y, int direction, u_short *p) + { + int leftDx = 0, leftDy = 0, rightDx = 0, rightDy = 0; + int forwardDx = 0, forwardDy = 0; + + int entries=0; + int distance=0; + + SetDirection(forwardDx, forwardDy, direction); + SetDirection(leftDx, leftDy, TurnLeft(direction)); + SetDirection(rightDx, rightDy, TurnRight(direction)); + + u_short *numEntries = p; + p++; + + for(;;) + { + x += forwardDx; + y += forwardDy; + + distance++; + + if(_mazeBitmap[MAZE_HEIGHT-1-(y)][x] == 0) // Did we run into wall? + { + *p++ = htons(MAZE_WALL); + *p++ = htons(distance); + entries++; + break; + } + + if(_mazeBitmap[MAZE_HEIGHT-1-(y+leftDy)][x+leftDx] != 0) + { + *p++ = htons(MAZE_EXIT_LEFT); + *p++ = htons(distance); + distance=0; + entries++; + } + + if(_mazeBitmap[MAZE_HEIGHT-1-(y+rightDy)][x+rightDx] != 0) + { + *p++ = htons(MAZE_EXIT_RIGHT); + *p++ = htons(distance); + distance=0; + entries++; + } + } + + *numEntries = htons(entries); + + return sizeof(u_short) * ((entries*2)+1); + } +}; + + + +bool MazeServerConnection::ReadEvent () +{ + bool startSending=false; + + int bytesRead = _socket->Read(_receivePtr, ReceiveBufferSpace()); + + if(bytesRead > 0) + { + _receivePtr = &_receivePtr[bytesRead]; + + if(ReceiveBufferBytes() >= (sizeof(u_short)*4)) + { + _receivePtr = reinterpret_cast<unsigned char *>(_receiveBuffer); + u_short *p = _receiveBuffer; + + if(ntohs(p[0]) == MAZE_QUERY_HALLWAY) + { + int x = ntohs(p[1]); + int y = ntohs(p[2]); + int direction = ntohs(p[3]); + + _sendPtr = reinterpret_cast<unsigned char *>(_sendBuffer); + _bytesToSend = _server->Read(x, y, direction, + static_cast<u_short *>(_sendBuffer)); + + startSending = true; + } + } + } + else + { + _shouldFree = true; + _server->_app->Progress(true, "Closing connection"); + } + + return startSending; +} + +bool MazeServerConnection::WriteEvent () +{ + bool sendMoreLater=false; + + if(_bytesToSend > 0) + { + int bytesWritten= _socket->Write(_sendPtr, _bytesToSend); + + if(bytesWritten > 0) + { + _bytesToSend -= bytesWritten; + _sendPtr = &_sendPtr[bytesWritten]; + } + else + { + _server->_app->Progress(false, + "Error writing %d bytes to socket", _bytesToSend); + } + + sendMoreLater = _bytesToSend > 0; + } + + return sendMoreLater; +} + + + + +class MazeClient : public MazeServices +{ +private: + MazeClient(const MazeClient&); + MazeClient& operator=(const MazeClient&); + + bool _visitedPoints[MAZE_WIDTH][MAZE_HEIGHT]; + u_short _searchTreeBuffer[SEARCHBUF_SIZE]; + u_short *_bufferPosition; + bool _foundExit; + FastOS_Socket *_sock; + BaseTest *_app; + +public: + MazeClient(BaseTest *app, FastOS_Socket *sock) + : MazeServices(), + _bufferPosition(_searchTreeBuffer), + _foundExit(false), + _sock(sock), + _app(app) + { + memset(_visitedPoints, 0, sizeof(_visitedPoints)); + } + + void Run () + { + int x = MAZE_START_X; + int y = MAZE_START_Y; + + Search(x, y, 1); + } + +#if 0 + void PrintOut() + { + if(_server != NULL) + { + for(int y=0; y<MAZE_HEIGHT; y++) + { + for(int x=0; x<MAZE_WIDTH; x++) + { + if(_server->_mazeBitmap[MAZE_HEIGHT-1-y][x] == 0) + printf("*"); + else if(_server->_mazeBitmap2[MAZE_HEIGHT-1-y][x] == 200) + printf("X"); + else + printf("%c", _server->_mazeBitmap2[MAZE_HEIGHT-1-y][x] == 50 ? '.' : ' '); + } + printf("\n"); + } + } + } +#endif + +#if 0 + void MarkPath (int x, int y, int direction, int length) + { + if(_server != NULL) + { + int dx, dy; + SetDirection(dx, dy, direction); + + while(length > 0) + { + x += dx; + y += dy; + + _server->_mazeBitmap2[MAZE_HEIGHT-1-y][x] = 200; + length--; + } + } + } +#endif + + bool Move (int &x, int &y, int direction, int length) + { + int dx = 0, dy = 0; + + int startx = x; + int starty = y; + + SetDirection(dx, dy, direction); + + bool continueAfterMove=true; + + while(length > 0) + { + x += dx; + y += dy; + + if((x == MAZE_END_X) && (y == MAZE_END_Y)) + { + _app->Progress(true, "Found exit at (%d, %d).", x, y); + _foundExit = true; + continueAfterMove = false; + break; + } + + if(_visitedPoints[x][y]) + { + continueAfterMove = false; + break; + } + else + { + _visitedPoints[x][y] = true; + } + length--; + } + + if(!continueAfterMove) + { + x = startx; + y = starty; + } + + return continueAfterMove; + } + + int ReadFromServer (int x, int y, int direction, u_short *p) + { + int readItems=0; + + + u_short *writePtr=p; + + *writePtr++ = htons(MAZE_QUERY_HALLWAY); + *writePtr++ = htons(static_cast<u_short>(x)); + *writePtr++ = htons(static_cast<u_short>(y)); + *writePtr++ = htons(static_cast<u_short>(direction)); + + int sendBytes = static_cast<int> + (reinterpret_cast<char *>(writePtr) - + reinterpret_cast<char *>(p)); + + int actualSent = _sock->Write(p, sendBytes); + + if(actualSent != sendBytes) + { + _app->Progress(false, "Sending %d bytes to maze server (rc=%d)", + sendBytes, actualSent); + } + else + { + int actualRead = _sock->Read(p, sizeof(u_short)); + + if(actualRead != sizeof(u_short)) + { + _app->Progress(false, "Reading %d bytes from maze server (rc=%d)", + sizeof(u_short), actualRead); + } + else + { + int packetSize = ntohs(*p); + p++; + int readBytes = packetSize * 2 * sizeof(u_short); + + actualRead = _sock->Read(p, readBytes); + + if(actualRead != readBytes) + { + _app->Progress(false, "Reading %d bytes from maze server (rc=%d)", + readBytes, actualRead); + } + + readItems = 1 + actualRead/sizeof(u_short); + } + } + + return readItems; + } + + void Search (int startX, int startY, int direction); +}; + + +void MazeClient::Search (int startX, int startY, int direction) +{ + u_short *p = _bufferPosition; + + int readEntries = ReadFromServer(startX, startY, direction, p); + + p++; + + _bufferPosition = &_bufferPosition[readEntries]; + assert(_bufferPosition < &_searchTreeBuffer[SEARCHBUF_SIZE]); + + bool continueSearching = true; + + int x=startX; + int y=startY; + int length=0; + + while(continueSearching) + { + u_short code = ntohs(*p++); + u_short distance = ntohs(*p++); + + switch(code) + { + case MAZE_WALL: + { + distance--; + // Make sure we have visited all the points + Move(x, y, direction, distance); + continueSearching = false; + break; + } + + case MAZE_EXIT_LEFT: + { + if(Move(x, y, direction, distance)) + Search(x, y, TurnLeft(direction)); + break; + } + + case MAZE_EXIT_RIGHT: + { + if(Move(x, y, direction, distance)) + Search(x, y, TurnRight(direction)); + break; + } + + default: + { + _app->Progress(false, "Unknown maze code (%d, %d) in packet", code, distance); + continueSearching = false; + break; + } + } + + length += distance; + + if(_foundExit) + { + continueSearching = false; + } + } +} + + +class SocketTest : public BaseTest +{ +public: + void StrictBindTest () + { + bool rc; + + TestHeader("Strict Bind Test"); + + // Fallback to localhost if we can't get the hostname + std::string strictBindHost("localhost"); + std::string hostName = FastOS_Socket::getHostName(); + if(!hostName.empty()) + strictBindHost = hostName; + + FastOS_ServerSocket *serverSocket = + new FastOS_ServerSocket(18333, 5, NULL, strictBindHost.c_str()); + + Progress(serverSocket != NULL, "Allocating serversocket instance"); + + rc = serverSocket->GetValidAddressFlag(); + Progress(rc, "Address Valid Flag check"); + + if(rc) + { + Progress(rc = serverSocket->Listen(), + "Strict bind socket to %s on port %d. Listen " + "for incoming connections.", strictBindHost.c_str(), 18333); + } + + delete(serverSocket); + Progress(true, "Deleted serversocket"); + + PrintSeparator(); + } + + void HttpClientTest () + { + bool rc=false; + + TestHeader("HTTP Client Test"); + + FastOS_Socket *sock = new FastOS_Socket(); + Progress(sock != NULL, "Allocating socket instance"); + + char hostAddress[] = "www.vg.no"; + + rc = sock->SetAddress(80, hostAddress); + Progress(rc, "Setting hostAddress (%s)", hostAddress); + + if(rc) + Progress(rc = sock->Connect(), "Connecting to %s", hostAddress); + + if(rc) + { + int localPort = sock->GetLocalPort(); + + Progress(localPort != -1, "Localport = %d", localPort); + + char sendCommand[] = "GET / HTTP/1.1\r\nHost: www.vg.no\r\n\r\n"; + int sendLength = strlen(sendCommand)+1; + + int wroteBytes = sock->Write(sendCommand, sendLength); + Progress(rc = (wroteBytes == sendLength), + "Write %d bytes to socket (GET / HTTP/1.1 ...)", wroteBytes); + + if(rc) + { + char expectedResult[] = "HTTP/1.X 200 Ok"; + + int readLength = strlen(expectedResult); + char *readBuffer = new char[readLength+1]; + + if(readBuffer != NULL) + { + memset(readBuffer, 0, readLength+1); + + int actualRead = sock->Read(readBuffer, readLength); + Progress(rc = (actualRead == readLength), "Read %d bytes from socket", actualRead); + Progress(true, "Contents: [%s]", readBuffer); + + expectedResult[7] = '0'; + rc = (strcasecmp(expectedResult, readBuffer) == 0); + expectedResult[7] = '1'; + rc |= (strcasecmp(expectedResult, readBuffer) == 0); + expectedResult[7] = '2'; + rc |= (strcasecmp(expectedResult, readBuffer) == 0); + + expectedResult[7] = 'X'; + + Progress(rc, "Comparing read result to expected result (%s)", expectedResult); + + delete [] readBuffer; + } + else + Fail("Allocating read buffer"); + } + + Progress(sock->Shutdown(), "Socket shutdown"); + Progress(rc = sock->Close(), "Closing socket"); + } + + delete(sock); + Progress(true, "Deleted socket"); + + PrintSeparator(); + } + + void ClientServerTest () + { + bool rc; + + TestHeader("Client/Server Test"); + + FastOS_ServerSocket *serverSocket = new FastOS_ServerSocket(18333); + Progress(serverSocket != NULL, "Allocating serversocket instance"); + + Progress(rc = serverSocket->Listen(), "Bind socket to port %d. Listen for incoming connections.", 18333); + assert(rc); + + delete(serverSocket); + Progress(true, "Deleted serversocket"); + + PrintSeparator(); + } + + + void MazeTest (char *serverAddress) + { + TestHeader("Maze Test"); + + bool rc; + FastOS_Socket *sock = new FastOS_Socket(); + + Progress(rc = (sock != NULL), "Allocating socket instance"); + if(rc) + { + sock->SetAddress(8001, serverAddress); + Progress(true, "Setting hostAddress (%s)", serverAddress); + + Progress(rc = sock->Connect(), "Connecting to %s", serverAddress); + if(rc) + { + MazeClient *client = new MazeClient(this, sock); + + Progress(rc = (client != NULL), "Allocating MazeClient instance"); + if(rc) + { + client->Run(); + delete(client); + } + } + + delete(sock); + } + + PrintSeparator(); + } + + void DoMazeServer () + { + TestHeader("Maze Server"); + + MazeServer *server = new MazeServer(this); + server->Run(); + + PrintSeparator(); + } + + void HostNameTest () + { + std::string errorMessage; + std::string hostName; + + TestHeader("Hostname Test"); + + hostName = FastOS_Socket::getHostName(&errorMessage); + + if(!hostName.empty()) + { + Progress(true, "Got hostname: [%s]", hostName.c_str()); + + + } + else + { + Progress(false, "Error getting hostname:\n%s", errorMessage.c_str()); + } + + PrintSeparator(); + } + + void HostAddressTest (bool testWithFailure) + { + std::string errorMessage; + + if (!testWithFailure) + { + TestHeader("Hostaddress Test"); + } + else + { + TestHeader("Provoked Hostaddress Failure Test"); + } + + std::string hostName; + + hostName = FastOS_Socket::getHostName(&errorMessage); + + if (!hostName.empty()) + { + if (testWithFailure) + { + Progress(true, "Got FQ Hostname: [%s], but will use ZZZZ instead", + hostName.c_str()); + hostName = "ZZZZ"; + } + else + { + Progress(true, "Got FQ Hostname: [%s]", + hostName.c_str()); + } + } + else + { + Progress(false, "Error getting hostname:\n%s", errorMessage.c_str()); + } + + PrintSeparator(); + } + + int Main () + { + printf("This test should be run in the 'test/workarea' directory.\n\n"); + printf("grep for the string '%s' to detect failures.\n\n", failString); + +#if DO_MAZE_SERVER + DoMazeServer(); +#else + char *mazeServerAddress = NULL; + + if(_argc == 3) + { + if(strcmp("/mazeserver", _argv[1]) == 0) + { + mazeServerAddress = _argv[2]; + } + } + + HostNameTest(); + HostAddressTest(false); + HostAddressTest(true); + HttpClientTest(); + ClientServerTest(); + StrictBindTest(); + + if(mazeServerAddress != NULL) + MazeTest(mazeServerAddress); +#endif + + + PrintSeparator(); + + printf("END OF TEST (%s)\n", _argv[0]); + + return 0; + } +}; + + +int main (int argc, char **argv) +{ + SocketTest app; + + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} diff --git a/fastos/src/tests/tests.h b/fastos/src/tests/tests.h new file mode 100644 index 00000000000..fdaa719669d --- /dev/null +++ b/fastos/src/tests/tests.h @@ -0,0 +1,174 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdio.h> +#include <string.h> + +#include <vespa/fastos/app.h> + +class BaseTest : public FastOS_Application +{ +private: + BaseTest(const BaseTest&); + BaseTest &operator=(const BaseTest&); + + int totallen; +public: + const char *okString; + const char *failString; + + BaseTest () + : totallen(70), + okString("SUCCESS"), + failString("FAILURE") + { + } + + virtual ~BaseTest() {}; + + void PrintSeparator () + { + for(int i=0; i<totallen; i++) printf("-"); + printf("\n"); + } + + virtual void PrintProgress (char *string) + { + printf(string); + } +#define MAX_STR_LEN 3000 + bool Progress (bool result, const char *str) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: %s\n", + result ? okString : failString, str); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, int d1) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, d1); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, int d1, int d2) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, d1, d2); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, const char *s1) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, s1); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, const FastOS_ThreadInterface *s1) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, s1); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, const FastOS_Socket *s1) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, s1); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, const char *s1, const char *s2) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, s1, s2); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, const char *s1, int d1) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, s1, d1); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool Progress (bool result, const char *str, int d1, const char *s1) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, d1, s1); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool ProgressI64 (bool result, const char *str, int64_t val) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, val); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + bool ProgressFloat (bool result, const char *str, float val) + { + char string[MAX_STR_LEN]; + sprintf(string, "%s: ", result ? okString : failString); + sprintf(&string[strlen(string)], str, val); + sprintf(&string[strlen(string)], "\n"); + PrintProgress(string); + return result; + } + + void Ok (const char *string) + { + Progress(true, string); + } + + void Fail (const char *string) + { + Progress(false, string); + } + + void TestHeader (const char *string) + { + int len = strlen(string); + int leftspace = (totallen - len)/2 - 2; + int rightspace = totallen - 4 - len - leftspace; + int i; + + printf("\n\n"); + for(i=0; i<totallen; i++) printf("*"); + printf("\n**"); + for(i=0; i<leftspace; i++) printf(" "); //forgot printf-specifier.. + printf(string); + for(i=0; i<rightspace; i++) printf(" "); + printf("**\n"); + for(i=0; i<totallen; i++) printf("*"); + printf("\n"); + } +}; diff --git a/fastos/src/tests/threadtest.cpp b/fastos/src/tests/threadtest.cpp new file mode 100644 index 00000000000..43ef6aef82a --- /dev/null +++ b/fastos/src/tests/threadtest.cpp @@ -0,0 +1,1466 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> + +#include <vespa/fastos/fastos.h> +#include "tests.h" + +#define PRI_TIME_COUNT 4096 + +enum JobCode +{ + PRINT_MESSAGE_AND_WAIT3SEC, + INCREASE_NUMBER, + PRIORITY_TEST, + WAIT_FOR_BREAK_FLAG, + WAIT_FOR_THREAD_TO_FINISH, + WAIT_FOR_CONDITION, + BOUNCE_CONDITIONS, + TEST_ID, + WAIT2SEC_AND_SIGNALCOND, + HOLD_MUTEX_FOR2SEC, + WAIT_2_SEC, + SILENTNOP, + NOP +}; + +class Job +{ +private: + Job(const Job &); + Job &operator=(const Job&); + +public: + JobCode code; + char *message; + FastOS_Mutex *mutex; + FastOS_Cond *condition; + FastOS_BoolCond *boolcondition; + FastOS_ThreadInterface *otherThread, *ownThread; + double *timebuf; + double average; + int result; + FastOS_Thread::Priority threadPri; + FastOS_ThreadId _threadId; + Job *otherjob; + int bouncewakeupcnt; + bool bouncewakeup; + + Job() + : code(NOP), + message(NULL), + mutex(NULL), + condition(NULL), + boolcondition(NULL), + otherThread(NULL), + ownThread(NULL), + timebuf(NULL), + average(0.0), + result(-1), + threadPri(FastOS_Thread::PRIORITY_NORMAL), + _threadId(), + otherjob(NULL), + bouncewakeupcnt(0), + bouncewakeup(false) + { + } + + ~Job() + { + if(message != NULL) + free(message); + } +}; + +static volatile int64_t number; +#define INCREASE_NUMBER_AMOUNT 10000 + +#define MUTEX_TEST_THREADS 6 +#define MAX_THREADS 7 + + +class ThreadTest : public BaseTest, public FastOS_Runnable +{ +private: + FastOS_Mutex printMutex; + +public: + ThreadTest(void) + : printMutex() + { + } + virtual ~ThreadTest() {}; + + void PrintProgress (char *string) + { + printMutex.Lock(); + BaseTest::PrintProgress(string); + printMutex.Unlock(); + } + + void Run (FastOS_ThreadInterface *thread, void *arg); + + void WaitForThreadsToFinish (Job *jobs, int count) + { + int i; + + Progress(true, "Waiting for threads to finish..."); + for(;;) + { + bool threadsFinished=true; + + for(i=0; i<count; i++) + { + if(jobs[i].result == -1) + { + threadsFinished = false; + break; + } + } + + FastOS_Thread::Sleep(500); + + if(threadsFinished) + break; + } + + Progress(true, "Threads finished"); + } + + void MutexTest (bool usingMutex) + { + if(usingMutex) + TestHeader("Mutex Test"); + else + TestHeader("Not Using Mutex Test"); + + + FastOS_ThreadPool *pool = new FastOS_ThreadPool(128*1024, MAX_THREADS); + + if(Progress(pool != NULL, "Allocating ThreadPool")) + { + int i; + Job jobs[MUTEX_TEST_THREADS]; + FastOS_Mutex *myMutex=NULL; + + if(usingMutex) + myMutex = new FastOS_Mutex(); + + for(i=0; i<MUTEX_TEST_THREADS; i++) + { + jobs[i].code = INCREASE_NUMBER; + jobs[i].mutex = myMutex; + } + + number = 0; + + for(i=0; i<MUTEX_TEST_THREADS; i++) + { + bool rc = (NULL != pool->NewThread(this, + static_cast<void *>(&jobs[i]))); + Progress(rc, "Creating Thread with%s mutex", (usingMutex ? "" : "out")); + }; + + WaitForThreadsToFinish(jobs, MUTEX_TEST_THREADS); + + + for(i=0; i<MUTEX_TEST_THREADS; i++) + { + Progress(true, "Thread returned with resultcode %d", jobs[i].result); + } + + bool wasOk=true; + int concurrentHits=0; + + for(i=0; i<MUTEX_TEST_THREADS; i++) + { + int result = jobs[i].result; + + if(usingMutex) + { + if((result % INCREASE_NUMBER_AMOUNT) != 0) + { + wasOk = false; + Progress(false, "Mutex locking did not work (%d).", result); + break; + } + } + else + { + if((result != 0) && + (result != INCREASE_NUMBER_AMOUNT*MUTEX_TEST_THREADS) && + (result % INCREASE_NUMBER_AMOUNT) == 0) + { + if((++concurrentHits) == 2) + { + wasOk = false; + Progress(false, "Very unlikely that threads are running " + "concurrently (%d)", jobs[i].result); + break; + } + } + } + } + + if(wasOk) + { + if(usingMutex) + { + Progress(true, "Using the mutex, the returned numbers were alligned."); + } + else + { + Progress(true, "Returned numbers were not alligned. " + "This was the expected result."); + } + } + + if(myMutex != NULL) + delete(myMutex); + + Progress(true, "Closing threadpool..."); + pool->Close(); + + Progress(true, "Deleting threadpool..."); + delete(pool); + } + PrintSeparator(); + } + + + void TooManyThreadsTest () + { + TestHeader("Too Many Threads Test"); + + FastOS_ThreadPool *pool = new FastOS_ThreadPool(128*1024, MAX_THREADS); + + if(Progress(pool != NULL, "Allocating ThreadPool")) + { + int i; + Job jobs[MAX_THREADS]; + + for(i=0; i<MAX_THREADS; i++) + { + jobs[i].code = PRINT_MESSAGE_AND_WAIT3SEC; + jobs[i].message = static_cast<char *>(malloc(100)); + sprintf(jobs[i].message, "Thread %d invocation", i+1); + } + + for(i=0; i<MAX_THREADS+1; i++) + { + if(i==MAX_THREADS) + { + bool rc = (NULL == pool->NewThread(this, + static_cast<void *>(&jobs[0]))); + Progress(rc, "Creating too many threads should fail."); + } + else + { + bool rc = (NULL != pool->NewThread(this, + static_cast<void *>(&jobs[i]))); + Progress(rc, "Creating Thread"); + } + }; + + WaitForThreadsToFinish(jobs, MAX_THREADS); + + Progress(true, "Verifying result codes..."); + for(i=0; i<MAX_THREADS; i++) + { + Progress(jobs[i].result == + static_cast<int>(strlen(jobs[i].message)), + "Checking result code from thread (%d==%d)", + jobs[i].result, strlen(jobs[i].message)); + } + + Progress(true, "Closing threadpool..."); + pool->Close(); + + Progress(true, "Deleting threadpool..."); + delete(pool); + } + PrintSeparator(); + } + + + void HowManyThreadsTest () + { + #define HOW_MAX_THREADS (1024) + TestHeader("How Many Threads Test"); + + FastOS_ThreadPool *pool = new FastOS_ThreadPool(128*1024, HOW_MAX_THREADS); + + if(Progress(pool != NULL, "Allocating ThreadPool")) + { + int i; + Job jobs[HOW_MAX_THREADS]; + + for(i=0; i<HOW_MAX_THREADS; i++) + { + jobs[i].code = PRINT_MESSAGE_AND_WAIT3SEC; + jobs[i].message = static_cast<char *>(malloc(100)); + sprintf(jobs[i].message, "Thread %d invocation", i+1); + } + + for(i=0; i<HOW_MAX_THREADS; i++) + { + if(i==HOW_MAX_THREADS) + { + bool rc = (NULL == pool->NewThread(this, + static_cast<void *>(&jobs[0]))); + Progress(rc, "Creating too many threads should fail."); + } + else + { + bool rc = (NULL != pool->NewThread(this, + static_cast<void *>(&jobs[i]))); + Progress(rc, "Creating Thread"); + } + }; + + WaitForThreadsToFinish(jobs, HOW_MAX_THREADS); + + Progress(true, "Verifying result codes..."); + for(i=0; i<HOW_MAX_THREADS; i++) + { + Progress(jobs[i].result == + static_cast<int>(strlen(jobs[i].message)), + "Checking result code from thread (%d==%d)", + jobs[i].result, strlen(jobs[i].message)); + } + + Progress(true, "Closing threadpool..."); + pool->Close(); + + Progress(true, "Deleting threadpool..."); + delete(pool); + } + PrintSeparator(); + } + + void CreateSingleThread () + { + TestHeader("Create Single Thread Test"); + + FastOS_ThreadPool *pool = new FastOS_ThreadPool(128*1024); + + if(Progress(pool != NULL, "Allocating ThreadPool")) + { + bool rc = (NULL != pool->NewThread(this, NULL)); + Progress(rc, "Creating Thread"); + + Progress(true, "Sleeping 3 seconds"); + FastOS_Thread::Sleep(3000); + } + + Progress(true, "Closing threadpool..."); + pool->Close(); + + Progress(true, "Deleting threadpool..."); + delete(pool); + PrintSeparator(); + } + + void CreateSingleThreadAndJoin () + { + TestHeader("Create Single Thread And Join Test"); + + FastOS_ThreadPool *pool = new FastOS_ThreadPool(128*1024); + + if(Progress(pool != NULL, "Allocating ThreadPool")) + { + Job job; + + job.code = NOP; + job.result = -1; + + bool rc = (NULL != pool->NewThread(this, &job)); + Progress(rc, "Creating Thread"); + + WaitForThreadsToFinish(&job, 1); + } + + Progress(true, "Closing threadpool..."); + pool->Close(); + + Progress(true, "Deleting threadpool..."); + delete(pool); + PrintSeparator(); + } + + void ThreadCreatePerformance (bool silent, int count, int outercount) + { + int i; + int j; + bool rc; + int threadsok; + FastOS_Time starttime; + FastOS_Time endtime; + FastOS_Time usedtime; + + if (!silent) + TestHeader("Thread Create Performance"); + + FastOS_ThreadPool *pool = new FastOS_ThreadPool(128 * 1024); + + if (!silent) + Progress(pool != NULL, "Allocating ThreadPool"); + if (pool != NULL) { + Job *jobs = new Job[count]; + + threadsok = 0; + starttime.SetNow(); + for (i = 0; i < count; i++) { + jobs[i].code = SILENTNOP; + jobs[i].ownThread = pool->NewThread(this, &jobs[i]); + rc = (jobs[i].ownThread != NULL); + if (rc) + threadsok++; + } + + for (j = 0; j < outercount; j++) { + for (i = 0; i < count; i++) { + if (jobs[i].ownThread != NULL) + jobs[i].ownThread->Join(); + jobs[i].ownThread = pool->NewThread(this, &jobs[i]); + rc = (jobs[i].ownThread != NULL); + if (rc) + threadsok++; + } + } + for (i = 0; i < count; i++) { + if (jobs[i].ownThread != NULL) + jobs[i].ownThread->Join(); + } + endtime.SetNow(); + usedtime = endtime; + usedtime -= starttime; + + if (!silent) { + Progress(true, "Used time: %d.%03d", + usedtime.GetSeconds(), usedtime.GetMicroSeconds() / 1000); + + double timeused = usedtime.GetSeconds() + + static_cast<double>(usedtime.GetMicroSeconds()) / 1000000.0; + ProgressFloat(true, "Threads/s: %6.1f", + static_cast<float>(static_cast<double>(threadsok) / + timeused)); + } + if (threadsok != ((outercount + 1) * count)) + Progress(false, "Only started %d of %d threads", threadsok, + (outercount + 1) * count); + + if (!silent) + Progress(true, "Closing threadpool..."); + pool->Close(); + delete [] jobs; + + if (!silent) + Progress(true, "Deleting threadpool..."); + delete(pool); + if (!silent) + PrintSeparator(); + } + } + + void ClosePoolStability(void) { + int i; + TestHeader("ThreadPool close stability test"); + for (i = 0; i < 8000; i++) { + // Progress(true, "Creating pool iteration %d", i + 1); + ThreadCreatePerformance(true, 2, 1); + } + PrintSeparator(); + } + + + + void ClosePoolTest () + { + TestHeader("Close Pool Test"); + + FastOS_ThreadPool pool(128*1024); + const int closePoolThreads=9; + Job jobs[closePoolThreads]; + + number = 0; + + for(int i=0; i<closePoolThreads; i++) + { + jobs[i].code = INCREASE_NUMBER; + + bool rc = (NULL != pool.NewThread(this, + static_cast<void *>(&jobs[i]))); + Progress(rc, "Creating Thread %d", i+1); + } + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + PrintSeparator(); + } + + void BreakFlagTest () + { + TestHeader("BreakFlag Test"); + + FastOS_ThreadPool pool(128*1024); + + const int breakFlagThreads=4; + + Job jobs[breakFlagThreads]; + + for(int i=0; i<breakFlagThreads; i++) + { + jobs[i].code = WAIT_FOR_BREAK_FLAG; + + bool rc = (NULL != pool.NewThread(this, + static_cast<void *>(&jobs[i]))); + Progress(rc, "Creating Thread %d", i+1); + } + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + PrintSeparator(); + } + + void SingleThreadJoinWaitMultipleTest(int variant) + { + bool rc=false; + + char testName[300]; + + sprintf(testName, "Single Thread Join Wait Multiple Test %d", variant); + TestHeader(testName); + + FastOS_ThreadPool pool(128*1024); + + const int testThreads=5; + int lastThreadNum = testThreads-1; + int i; + + Job jobs[testThreads]; + + FastOS_Mutex jobMutex; + + // The mutex is used to pause the first threads until we have created + // the last one. + jobMutex.Lock(); + + for(i=0; i<lastThreadNum; i++) + { + jobs[i].code = WAIT_FOR_THREAD_TO_FINISH; + jobs[i].mutex = &jobMutex; + jobs[i].ownThread = pool.NewThread(this, + static_cast<void *>(&jobs[i])); + + rc = (jobs[i].ownThread != NULL); + Progress(rc, "Creating Thread %d", i+1); + + if(!rc) + break; + } + + if(rc) + { + jobs[lastThreadNum].code = (((variant & 2) != 0) ? NOP : PRINT_MESSAGE_AND_WAIT3SEC); + jobs[lastThreadNum].message = strdup("This is the thread that others wait for."); + + FastOS_ThreadInterface *lastThread; + + lastThread = pool.NewThread(this, + static_cast<void *> + (&jobs[lastThreadNum])); + + rc = (lastThread != NULL); + Progress(rc, "Creating last thread"); + + if(rc) + { + for(i=0; i<lastThreadNum; i++) + { + jobs[i].otherThread = lastThread; + } + } + } + + jobMutex.Unlock(); + + if((variant & 1) != 0) + { + for(i=0; i<lastThreadNum; i++) + { + Progress(true, "Waiting for thread %d to finish using Join()", i+1); + jobs[i].ownThread->Join(); + Progress(true, "Thread %d finished.", i+1); + } + } + + Progress(true, "Closing pool."); + pool.Close(); + Progress(true, "Pool closed."); + PrintSeparator(); + } + + void WaitForXThreadsToHaveWait (Job *jobs, + int jobCount, + FastOS_Cond *condition, + int numWait) + { + Progress(true, "Waiting for %d threads to be in wait state", numWait); + + int oldNumber=-10000; + for(;;) + { + int waitingThreads=0; + + condition->Lock(); + + for(int i=0; i<jobCount; i++) + { + if(jobs[i].result == 1) + waitingThreads++; + } + + condition->Unlock(); + + if(waitingThreads != oldNumber) + Progress(true, "%d threads are waiting", waitingThreads); + + oldNumber = waitingThreads; + + if(waitingThreads == numWait) + break; + + FastOS_Thread::Sleep(100); + } + } + + void SharedSignalAndBroadcastTest (Job *jobs, int numThreads, + FastOS_Cond *condition, + FastOS_ThreadPool *pool) + { + for(int i=0; i<numThreads; i++) + { + jobs[i].code = WAIT_FOR_CONDITION; + jobs[i].condition = condition; + jobs[i].ownThread = pool->NewThread(this, + static_cast<void *>(&jobs[i])); + + bool rc=(jobs[i].ownThread != NULL); + Progress(rc, "CreatingThread %d", i+1); + } + + WaitForXThreadsToHaveWait (jobs, numThreads, + condition, numThreads); + + // Threads are not guaranteed to have entered sleep yet, + // as this test only tests for result code + // Wait another second to be sure. + FastOS_Thread::Sleep(1000); + } + + void BounceTest(void) + { + TestHeader("Bounce Test"); + + FastOS_ThreadPool pool(128 * 1024); + FastOS_Cond cond1; + FastOS_Cond cond2; + Job job1; + Job job2; + FastOS_Time checkTime; + int cnt1; + int cnt2; + int cntsum; + int lastcntsum; + + job1.code = BOUNCE_CONDITIONS; + job2.code = BOUNCE_CONDITIONS; + job1.otherjob = &job2; + job2.otherjob = &job1; + job1.condition = &cond1; + job2.condition = &cond2; + + job1.ownThread = pool.NewThread(this, static_cast<void *>(&job1)); + job2.ownThread = pool.NewThread(this, static_cast<void *>(&job2)); + + lastcntsum = -1; + for (int iter = 0; iter < 8; iter++) { + checkTime.SetNow(); + + int left = static_cast<int>(checkTime.MilliSecsToNow()); + while (left < 1000) { + FastOS_Thread::Sleep(1000 - left); + left = static_cast<int>(checkTime.MilliSecsToNow()); + } + + cond1.Lock(); + cnt1 = job1.bouncewakeupcnt; + cond1.Unlock(); + cond2.Lock(); + cnt2 = job2.bouncewakeupcnt; + cond2.Unlock(); + cntsum = cnt1 + cnt2; + Progress(lastcntsum != cntsum, "%d bounces", cntsum); + lastcntsum = cntsum; + } + + job1.ownThread->SetBreakFlag(); + cond1.Lock(); + job1.bouncewakeup = true; + cond1.Signal(); + cond1.Unlock(); + + job2.ownThread->SetBreakFlag(); + cond2.Lock(); + job2.bouncewakeup = true; + cond2.Signal(); + cond2.Unlock(); + + pool.Close(); + Progress(true, "Pool closed."); + PrintSeparator(); + + } + + void SignalTest () + { + const int numThreads = 5; + + TestHeader("Signal Test"); + + FastOS_ThreadPool pool(128*1024); + Job jobs[numThreads]; + FastOS_Cond condition; + + SharedSignalAndBroadcastTest(jobs, numThreads, &condition, &pool); + + for(int i=0; i<numThreads; i++) + { + condition.Signal(); + WaitForXThreadsToHaveWait(jobs, numThreads, + &condition, numThreads-1-i); + } + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + PrintSeparator(); + } + + void BroadcastTest () + { + TestHeader("Broadcast Test"); + + const int numThreads = 5; + + FastOS_ThreadPool pool(128*1024); + Job jobs[numThreads]; + FastOS_Cond condition; + + SharedSignalAndBroadcastTest(jobs, numThreads, &condition, &pool); + + condition.Broadcast(); + WaitForXThreadsToHaveWait(jobs, numThreads, &condition, 0); + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + PrintSeparator(); + } + + void PriorityTest (int sign) + { + if(sign == 1) + TestHeader("Priority Test (positive)"); + else + TestHeader("Priority Test (negative)"); + + const int numThreads = 5; + + FastOS_ThreadPool pool(128*1024); + Job jobs[numThreads]; + + int i; + FastOS_BoolCond boolcondition; + double *timebuf = new double[numThreads * PRI_TIME_COUNT]; + + boolcondition.SetBusy(); + + for(i=0; i<numThreads; i++) + { + jobs[i].code = PRIORITY_TEST; + jobs[i].boolcondition = &boolcondition; + jobs[i].timebuf = &timebuf[i*PRI_TIME_COUNT]; + jobs[i].threadPri = static_cast<FastOS_Thread::Priority>((-2+i)*sign); + jobs[i].average = 0; + jobs[i].ownThread = pool.NewThread(this, + static_cast<void *>(&jobs[i])); + + bool rc=(jobs[i].ownThread != NULL); + Progress(rc, "CreatingThread %d", i+1); + } + + // Start testing + Progress(true, "Start testing..."); + boolcondition.ClearBusyBroadcast(); + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + + double firstTime, lastTime, totalTime; + firstTime = lastTime = timebuf[0]; + + for(i=0; i<(numThreads * PRI_TIME_COUNT); i++) + { + double timevalue = timebuf[i]; + + if(timevalue < firstTime) + firstTime = timevalue; + + if(timevalue > lastTime) + lastTime = timevalue; + + int job = (i/PRI_TIME_COUNT); + jobs[job].average += timevalue; + } + + totalTime = (lastTime - firstTime); + + for(i=0; i<numThreads; i++) + { + const int slen=45; + + char timestr[slen+1]; + memset(timestr, ' ', slen); + timestr[slen] = '\0'; + + double *p = &timebuf[i*PRI_TIME_COUNT]; + + for(int j=0; j<PRI_TIME_COUNT; j++) + { + int pos = static_cast<int> + (((*p++) - firstTime) * (slen-1) / totalTime); + timestr[pos] = '*'; + } + + Progress(true, "Pri %2d: %s", jobs[i].threadPri, timestr); + } + + char averagestr[80]; + + averagestr[0] = '\0'; + + for(i=0; i<numThreads; i++) + { + jobs[i].average /= PRI_TIME_COUNT; + + // convert average to percent + jobs[i].average = (jobs[i].average - firstTime) / totalTime; + + sprintf(&averagestr[strlen(averagestr)], "%.3f ", jobs[i].average); + } + Progress(true, "Percentages: %s", averagestr); + + bool okExecution=true; + + for(i=0; i<(numThreads-1); i++) + { + double val1 = jobs[i].average * sign; + double val2 = jobs[i+1].average * sign; + + if(val1 < val2) + { + okExecution = false; + break; + } + } + + Progress(okExecution, "Checking order of execution"); + delete [] timebuf; + + PrintSeparator(); + } + + void ThreadIdTest () + { + const int numThreads = 5; + int i,j; + + TestHeader ("Thread Id Test"); + + FastOS_ThreadPool pool(128*1024); + Job jobs[numThreads]; + FastOS_Mutex slowStartMutex; + + slowStartMutex.Lock(); // Halt all threads until we want them to run + + for(i=0; i<numThreads; i++) { + jobs[i].code = TEST_ID; + jobs[i].result = -1; + jobs[i]._threadId = 0; + jobs[i].mutex = &slowStartMutex; + jobs[i].ownThread = pool.NewThread(this, + static_cast<void *>(&jobs[i])); + bool rc=(jobs[i].ownThread != NULL); + if(rc) + jobs[i]._threadId = jobs[i].ownThread->GetThreadId(); + Progress(rc, "CreatingThread %d id:%lu", i+1, + (unsigned long)(jobs[i]._threadId)); + + for(j=0; j<i; j++) { + if(jobs[j]._threadId == jobs[i]._threadId) { + Progress(false, + "Two different threads received the " + "same thread id (%lu)", + (unsigned long)(jobs[i]._threadId)); + } + } + } + + slowStartMutex.Unlock(); // Allow threads to run + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + + for(i=0; i<numThreads; i++) { + Progress(jobs[i].result == 1, + "Thread %lu: ID comparison (current vs stored)", + (unsigned long)(jobs[i]._threadId)); + } + + PrintSeparator(); + } + + void TimedWaitTest () + { + TestHeader("Cond Timed Wait Test"); + + FastOS_ThreadPool pool(128*1024); + Job job; + FastOS_Cond condition; + + job.code = WAIT2SEC_AND_SIGNALCOND; + job.result = -1; + job.condition = &condition; + job.ownThread = pool.NewThread(this, + static_cast<void *>(&job)); + + Progress(job.ownThread !=NULL, "Creating thread"); + + if(job.ownThread != NULL) + { + condition.Lock(); + bool gotCond = condition.TimedWait(500); + Progress(!gotCond, "We should not get the condition just yet (%s)", + gotCond ? "got it" : "didn't get it"); + gotCond = condition.TimedWait(500); + Progress(!gotCond, "We should not get the condition just yet (%s)", + gotCond ? "got it" : "didn't get it"); + gotCond = condition.TimedWait(5000); + Progress(gotCond, "We should have got the condition now (%s)", + gotCond ? "got it" : "didn't get it"); + condition.Unlock(); + } + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + + PrintSeparator(); + } + + void TryLockTest () + { + TestHeader("Mutex TryLock Test"); + + FastOS_ThreadPool pool(128*1024); + Job job; + FastOS_Mutex mtx; + + job.code = HOLD_MUTEX_FOR2SEC; + job.result = -1; + job.mutex = &mtx; + job.ownThread = pool.NewThread(this, + static_cast<void *>(&job)); + + Progress(job.ownThread !=NULL, "Creating thread"); + + if(job.ownThread != NULL) + { + bool lockrc; + + FastOS_Thread::Sleep(1000); + + for(int i=0; i<5; i++) + { + lockrc = mtx.TryLock(); + Progress(!lockrc, "We should not get the mutex lock just yet (%s)", + lockrc ? "got it" : "didn't get it"); + if(lockrc) { + mtx.Unlock(); + break; + } + } + + FastOS_Thread::Sleep(2000); + + lockrc = mtx.TryLock(); + Progress(lockrc, "We should get the mutex lock now (%s)", + lockrc ? "got it" : "didn't get it"); + + if(lockrc) + mtx.Unlock(); + + Progress(true, "Attempting to do normal lock..."); + mtx.Lock(); + Progress(true, "Got lock. Attempt to do normal unlock..."); + mtx.Unlock(); + Progress(true, "Unlock OK."); + } + + Progress(true, "Waiting for threads to finish using pool.Close()..."); + pool.Close(); + Progress(true, "Pool closed."); + + PrintSeparator(); + } + + void ThreadStatsTest () + { + int inactiveThreads; + int activeThreads; + int startedThreads; + + TestHeader("Thread Statistics Test"); + + FastOS_ThreadPool pool(128*1024); + Job job[2]; + + inactiveThreads = pool.GetNumInactiveThreads(); + Progress(inactiveThreads == 0, "Initial inactive threads = %d", + inactiveThreads); + activeThreads = pool.GetNumActiveThreads(); + Progress(activeThreads == 0, "Initial active threads = %d", + activeThreads); + startedThreads = pool.GetNumStartedThreads(); + Progress(startedThreads == 0, "Initial started threads = %d", + startedThreads); + + job[0].code = WAIT_FOR_BREAK_FLAG; + job[0].ownThread = pool.NewThread(this, + static_cast<void *>(&job[0])); + + FastOS_Thread::Sleep(1000); + + inactiveThreads = pool.GetNumInactiveThreads(); + Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); + activeThreads = pool.GetNumActiveThreads(); + Progress(activeThreads == 1, "Active threads = %d", activeThreads); + startedThreads = pool.GetNumStartedThreads(); + Progress(startedThreads == 1, "Started threads = %d", startedThreads); + + job[1].code = WAIT_FOR_BREAK_FLAG; + job[1].ownThread = pool.NewThread(this, + static_cast<void *>(&job[1])); + + FastOS_Thread::Sleep(1000); + + inactiveThreads = pool.GetNumInactiveThreads(); + Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); + activeThreads = pool.GetNumActiveThreads(); + Progress(activeThreads == 2, "Active threads = %d", activeThreads); + startedThreads = pool.GetNumStartedThreads(); + Progress(startedThreads == 2, "Started threads = %d", startedThreads); + + Progress(true, "Setting breakflag on threads..."); + job[0].ownThread->SetBreakFlag(); + job[1].ownThread->SetBreakFlag(); + + FastOS_Thread::Sleep(3000); + + inactiveThreads = pool.GetNumInactiveThreads(); + Progress(inactiveThreads == 2, "Inactive threads = %d", inactiveThreads); + activeThreads = pool.GetNumActiveThreads(); + Progress(activeThreads == 0, "Active threads = %d", activeThreads); + startedThreads = pool.GetNumStartedThreads(); + Progress(startedThreads == 2, "Started threads = %d", startedThreads); + + + Progress(true, "Repeating process in the same pool..."); + + job[0].code = WAIT_FOR_BREAK_FLAG; + job[0].ownThread = pool.NewThread(this, static_cast<void *>(&job[0])); + + FastOS_Thread::Sleep(1000); + + inactiveThreads = pool.GetNumInactiveThreads(); + Progress(inactiveThreads == 1, "Inactive threads = %d", inactiveThreads); + activeThreads = pool.GetNumActiveThreads(); + Progress(activeThreads == 1, "Active threads = %d", activeThreads); + startedThreads = pool.GetNumStartedThreads(); + Progress(startedThreads == 3, "Started threads = %d", startedThreads); + + job[1].code = WAIT_FOR_BREAK_FLAG; + job[1].ownThread = pool.NewThread(this, static_cast<void *>(&job[1])); + + FastOS_Thread::Sleep(1000); + + inactiveThreads = pool.GetNumInactiveThreads(); + Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); + activeThreads = pool.GetNumActiveThreads(); + Progress(activeThreads == 2, "Active threads = %d", activeThreads); + startedThreads = pool.GetNumStartedThreads(); + Progress(startedThreads == 4, "Started threads = %d", startedThreads); + + Progress(true, "Setting breakflag on threads..."); + job[0].ownThread->SetBreakFlag(); + job[1].ownThread->SetBreakFlag(); + + FastOS_Thread::Sleep(3000); + + inactiveThreads = pool.GetNumInactiveThreads(); + Progress(inactiveThreads == 2, "Inactive threads = %d", inactiveThreads); + activeThreads = pool.GetNumActiveThreads(); + Progress(activeThreads == 0, "Active threads = %d", activeThreads); + startedThreads = pool.GetNumStartedThreads(); + Progress(startedThreads == 4, "Started threads = %d", startedThreads); + + + pool.Close(); + Progress(true, "Pool closed."); + + PrintSeparator(); + } + + int Main (); + + void LeakTest () + { + TestHeader("Leak Test"); + + int allocCount = 2 * 1024 * 1024; + int progressIndex= allocCount/8; + int i; + + for(i=0; i<allocCount; i++) + { + FastOS_Mutex *mtx = new FastOS_Mutex(); + mtx->Lock(); + mtx->Unlock(); + delete mtx; + + if((i % progressIndex) == (progressIndex - 1)) + Progress(true, "Tested %d FastOS_Mutex instances", i + 1); + } + + for(i=0; i<allocCount; i++) + { + FastOS_Cond *cond = new FastOS_Cond(); + delete cond; + + if((i % progressIndex) == (progressIndex - 1)) + Progress(true, "Tested %d FastOS_Cond instances", i+1); + } + + for(i=0; i<allocCount; i++) + { + FastOS_BoolCond *cond = new FastOS_BoolCond(); + delete cond; + + if((i % progressIndex) == (progressIndex - 1)) + Progress(true, "Tested %d FastOS_BoolCond instances", i+1); + } + + PrintSeparator(); + } + + void SynchronizationStressTest () + { + TestHeader("Synchronization Object Stress Test"); + + const int allocCount = 150000; + int i; + + FastOS_Mutex **mutexes = new FastOS_Mutex*[allocCount]; + + FastOS_Time startTime, nowTime; + startTime.SetNow(); + + for(i=0; i<allocCount; i++) + mutexes[i] = new FastOS_Mutex(); + + nowTime.SetNow(); + Progress(true, "Allocated %d mutexes at time: %d ms", allocCount, + static_cast<int>(nowTime.MilliSecs() - startTime.MilliSecs())); + + for(int e=0; e<4; e++) + { + for(i=0; i<allocCount; i++) + mutexes[i]->Lock(); + + for(i=0; i<allocCount; i++) + mutexes[i]->Unlock(); + + nowTime.SetNow(); + Progress(true, "Tested %d mutexes at time: %d ms", allocCount, + static_cast<int>(nowTime.MilliSecs() - + startTime.MilliSecs())); + } + for(i=0; i<allocCount; i++) + delete mutexes[i]; + + nowTime.SetNow(); + Progress(true, "Deleted %d mutexes at time: %d ms", allocCount, + static_cast<int>(nowTime.MilliSecs() - startTime.MilliSecs())); + + delete [] mutexes; + + PrintSeparator(); + } + +}; + +int ThreadTest::Main () +{ + printf("grep for the string '%s' to detect failures.\n\n", failString); + + // HowManyThreadsTest(); + SynchronizationStressTest(); + LeakTest(); + ThreadStatsTest(); + TryLockTest(); + TimedWaitTest(); + ThreadIdTest(); + PriorityTest(1); + PriorityTest(-1); + SignalTest(); + BroadcastTest(); + CreateSingleThread(); + CreateSingleThreadAndJoin(); + TooManyThreadsTest(); + MutexTest(false); + MutexTest(true); + ClosePoolTest(); + BreakFlagTest(); + SingleThreadJoinWaitMultipleTest(0); + SingleThreadJoinWaitMultipleTest(1); + SingleThreadJoinWaitMultipleTest(2); + SingleThreadJoinWaitMultipleTest(3); + SingleThreadJoinWaitMultipleTest(2); + SingleThreadJoinWaitMultipleTest(1); + SingleThreadJoinWaitMultipleTest(0); + BreakFlagTest(); + ClosePoolTest(); + MutexTest(true); + MutexTest(false); + TooManyThreadsTest(); + CreateSingleThreadAndJoin(); + CreateSingleThread(); + BroadcastTest(); + SignalTest(); + ThreadCreatePerformance(false, 500, 100); + ClosePoolStability(); + BounceTest(); + + printf("END OF TEST (%s)\n", _argv[0]); + + return 0; +} + +volatile int busyCnt; + +void ThreadTest::Run (FastOS_ThreadInterface *thread, void *arg) +{ + if(arg == NULL) + return; + + Job *job = static_cast<Job *>(arg); + char someStack[15*1024]; + + memset(someStack, 0, 15*1024); + + switch(job->code) + { + case PRIORITY_TEST: + { + thread->SetPriority(job->threadPri); + + job->boolcondition->WaitBusy(); + double *p = job->timebuf; + for(int i=0; i<PRI_TIME_COUNT; i++) + { + for(int j=0; j<8192; j++) + { + busyCnt = busyCnt*10; + } + + FastOS_Time now; + now.SetNow(); + + *p++ = now.MilliSecs(); + } + break; + } + + case SILENTNOP: + { + job->result = 1; + break; + } + + case NOP: + { + Progress(true, "Doing NOP"); + job->result = 1; + break; + } + + case PRINT_MESSAGE_AND_WAIT3SEC: + { + Progress(true, "Thread printing message: [%s]", job->message); + job->result = strlen(job->message); + + FastOS_Thread::Sleep(3000); + break; + } + + case INCREASE_NUMBER: + { + int result; + + if(job->mutex != NULL) + job->mutex->Lock(); + + result = static_cast<int>(number); + + int sleepOn = (INCREASE_NUMBER_AMOUNT/2) * 321/10000; + for(int i=0; i<(INCREASE_NUMBER_AMOUNT/2); i++) + { + number = number + 2; + + if(i == sleepOn) + FastOS_Thread::Sleep(1000); + } + + if(job->mutex != NULL) + job->mutex->Unlock(); + + job->result = result; // This marks the end of the thread + + break; + } + + case WAIT_FOR_BREAK_FLAG: + { + for(;;) + { + FastOS_Thread::Sleep(1000); + + if(thread->GetBreakFlag()) + { + Progress(true, "Thread %p got breakflag", thread); + break; + } + } + break; + } + + case WAIT_FOR_THREAD_TO_FINISH: + { + if(job->mutex) + job->mutex->Lock(); + + if(job->otherThread != NULL) + job->otherThread->Join(); + + if(job->mutex) + job->mutex->Unlock(); + break; + } + + case WAIT_FOR_CONDITION: + { + job->condition->Lock(); + + job->result = 1; + + job->condition->Wait(); + job->condition->Unlock(); + + job->result = 0; + + break; + } + + case BOUNCE_CONDITIONS: + { + while (!thread->GetBreakFlag()) { + job->otherjob->condition->Lock(); + job->otherjob->bouncewakeupcnt++; + job->otherjob->bouncewakeup = true; + job->otherjob->condition->Signal(); + job->otherjob->condition->Unlock(); + + job->condition->Lock(); + while (!job->bouncewakeup) + job->condition->TimedWait(1); + job->bouncewakeup = false; + job->condition->Unlock(); + } + break; + } + + case TEST_ID: + { + job->mutex->Lock(); // Initially the parent threads owns the lock + job->mutex->Unlock(); // It is unlocked when we should start + + FastOS_ThreadId currentId = FastOS_Thread::GetCurrentThreadId(); + + if(currentId == job->_threadId) + job->result = 1; + else + job->result = -1; + break; + } + + case WAIT2SEC_AND_SIGNALCOND: + { + FastOS_Thread::Sleep(2000); + job->condition->Signal(); + job->result = 1; + break; + } + + case HOLD_MUTEX_FOR2SEC: + { + job->mutex->Lock(); + FastOS_Thread::Sleep(2000); + job->mutex->Unlock(); + job->result = 1; + break; + } + + case WAIT_2_SEC: + { + FastOS_Thread::Sleep(2000); + job->result = 1; + break; + } + + default: + Progress(false, "Unknown jobcode"); + break; + } +} + +int main (int argc, char **argv) +{ + ThreadTest app; + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} diff --git a/fastos/src/tests/timetest.cpp b/fastos/src/tests/timetest.cpp new file mode 100644 index 00000000000..b787914ad89 --- /dev/null +++ b/fastos/src/tests/timetest.cpp @@ -0,0 +1,297 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <math.h> + +#include <vespa/fastos/fastos.h> +#include "tests.h" + +using namespace fastos; + +class TimeTest : public BaseTest +{ +private: + +public: + virtual ~TimeTest() {}; + + void Wait3SecondsTest () + { + TestHeader("Wait 3 seconds test"); + bool rc=false; + FastOS_Time before, timing; + + Progress(true, "Waiting 3 seconds..."); + before.SetNow(); + + FastOS_Thread::Sleep(3000); + + timing.SetNow(); + + rc = (timing > before); + Progress(rc, "AfterTime > BeforeTime"); + + rc = (before < timing); + Progress(rc, "BeforeTime < AfterTime"); + + rc = (before <= timing); + Progress(rc, "BeforeTime <= AfterTime"); + + rc = (timing >= before); + Progress(rc, "AfterTime >= BeforeTime"); + + rc = (timing >= timing); + Progress(rc, "AfterTime >= AfterTime"); + + rc = (timing <= timing); + Progress(rc, "AfterTime <= AfterTime"); + + FastOS_Time copyOfAfterTime = timing; + rc = (copyOfAfterTime == timing); + Progress(rc, "CopyOfAfterTime == AfterTime"); + + timing -= before; + + double milliseconds = timing.MilliSecs(); + rc = ((milliseconds >= 1900) && + (milliseconds <= 10000)); // More than 10 seconds?? + + Progress(rc, "Waittime = %d milliseconds", + static_cast<int>(milliseconds)); + + double microseconds = timing.MicroSecs(); + rc = ((microseconds >= 1900000) && + (microseconds <= 10000000)); // More than 10 seconds?? + + Progress(rc, "Waittime = %d microseconds", + static_cast<int>(microseconds)); + + timing += before; + rc = (copyOfAfterTime == timing); + Progress(rc, "CopyOfAfterTime == AfterTime (after minus-plus)"); + + PrintSeparator(); + } + + void TimeArithmeticTest () + { + bool rc=false; + + TestHeader("Other Time Aritmetic Test"); + + FastOS_Time time1, time2; + time1.SetZero(); + time2.SetZero(); + + rc = (time1 == time2); + Progress(rc, "Two zero times are equal"); + + time1.SetMilliSecs(124); + Progress(long(time1.MilliSecs()) == 124, + "SetMilliSecs(124) -> MilliSecs(%ld)", + long(time1.MilliSecs())); + + time1.SetMicroSecs(123000); + Progress(long(time1.MicroSecs()) == 123000, + "SetMicroSecs(123000) -> MicroSecs(%ld)", + long(time1.MicroSecs())); + + time1.SetMilliSecs(999124); + Progress(long(time1.MilliSecs()) == 999124, + "SetMilliSecs(999124) -> MilliSecs(%ld)", + long(time1.MilliSecs())); + + time1.SetMicroSecs(9123000); + Progress(long(time1.MicroSecs()) == 9123000, + "SetMicroSecs(9123000) -> MicroSecs(%ld)", + long(time1.MicroSecs())); + + time2 = time1; + Progress(long(time2.MicroSecs()) == 9123000, + "[time2 = time1] -> time2.MicroSecs(%ld)", + long(time2.MicroSecs())); + + time2 += time1; + Progress(long(time2.MicroSecs()) == 9123000*2, + "[time2 += time1] -> time2.MicroSecs(%ld)", + long(time2.MicroSecs())); + + time2 += time2; + Progress(long(time2.MicroSecs()) == 9123000*4, + "[time2 += time2] -> time2.MicroSecs(%ld)", + long(time2.MicroSecs())); + + time2 -= time1; + Progress(long(time2.MicroSecs()) == 9123000*3, + "[time2 -= time2] -> time2.MicroSecs(%ld)", + long(time2.MicroSecs())); + + Progress(time2 > time1, + "[time2 > time2] -> %s", time2 > time1 ? "true" : "false"); + Progress(time2 >= time1, + "[time2 >= time2] -> %s", time2 >= time1 ? "true" : "false"); + Progress(time1 < time2, + "[time1 < time2] -> %s", time1 < time2 ? "true" : "false"); + Progress(time1 <= time2, + "[time1 <= time2] -> %s", time1 <= time2 ? "true" : "false"); + + Progress(!(time2 < time1), + "[time2 < time2] -> %s", time2 < time1 ? "true" : "false"); + Progress(!(time2 <= time1), + "[time2 <= time1] -> %s", time2 <= time1 ? "true" : "false"); + Progress(!(time1 > time2), + "[time1 > time2] -> %s", time1 > time2 ? "true" : "false"); + Progress(!(time1 >= time2), + "[time1 >= time2] -> %s", time1 >= time2 ? "true" : "false"); + Progress(!(time1 == time2), + "[time1 == time2] -> %s", time1 == time2 ? "true" : "false"); + + time1 = time2; + Progress(long(time1.MicroSecs()) == 9123000*3, + "[time1 = time2] -> time1.MicroSecs(%ld)", + long(time1.MicroSecs())); + Progress(time2 >= time1, + "[time2 >= time2] -> %s", time2 >= time1 ? "true" : "false"); + Progress(time1 <= time2, + "[time1 <= time2] -> %s", time1 <= time2 ? "true" : "false"); + Progress((time1 == time2), + "[time1 == time2] -> %s", time1 == time2 ? "true" : "false"); + + time1.SetZero(); + Progress(long(time1.MilliSecs()) == 0, + "SetZero() -> MilliSecs(%ld)", + long(time1.MilliSecs())); + + Progress(long(time1.MicroSecs()) == 0, + "SetZero() -> MicroSecs(%ld)", + long(time1.MicroSecs())); + + time1 = 2.5; + Progress(long(time1.MilliSecs()) == 2500, + "time2 = 2.5 -> MilliSecs(%ld)", + long(time1.MilliSecs())); + + time2 = 3.9; + Progress(long(time2.MicroSecs()) == 3900000, + "time2 = 3.9 -> MicroSecs(%ld)", + long(time2.MicroSecs())); + + time1.SetNow(); + FastOS_Thread::Sleep(1000); + double waited = time1.MicroSecsToNow(); + Progress((waited >= 950000) && (waited <= 1200000), + "Slept 1000 ms, MicroSecsToNow(%ld)", long(waited)); + + time2.SetNow(); + FastOS_Thread::Sleep(2000); + waited = time2.MilliSecsToNow(); + Progress((waited >= (2*950)) && (waited <= (2*1200)), + "Slept 2000 ms, MilliSecsToNow(%ld)", long(waited)); + + time2.SetMicroSecs(40000); + time2.AddMicroSecs(1000000); + Progress(long(time2.MicroSecs()) == (40000 + 1000000), + "[SetMicroSecs(40000); AddMicroSecs(1000000)] -> MicroSecs(%ld)", + long(time2.MicroSecs())); + + time1.SetMicroSecs(9123000); + time1.SubtractMicroSecs(512000); + Progress(long(time1.MicroSecs()) == (9123000 - 512000), + "[SetMicroSecs(9123000); SubMicroSecs(512000)] -> MicroSecs(%ld)", + long(time1.MicroSecs())); + + time1.SetMilliSecs(400); + time1.AddMilliSecs(1000001); + Progress(long(time1.MilliSecs()) == (400 + 1000001), + "[SetMilliSecs(400); AddMilliSecs(1000001)] -> MilliSecs(%ld)", + long(time1.MilliSecs())); + + time2.SetMilliSecs(9123213); + time2.SubtractMilliSecs(512343); + Progress(long(time2.MilliSecs()) == (9123213 - 512343), + "[SetMilliSecs(9123213); SubMilliSecs(512343)] -> MilliSecs(%ld)", + long(time2.MilliSecs())); + + Progress(time2.GetSeconds() == ((9123213 - 512343) / (1000)), + "[time2.GetSeconds()] -> %ld", time2.GetSeconds()); + Progress(int64_t(time2.GetMicroSeconds()) == ((int64_t(9123213 - 512343)*1000) % (1000000)), + "[time2.GetMicroSeconds()] -> %ld", time2.GetMicroSeconds()); + + PrintSeparator(); + } + + void TimeStepTest () + { + TestHeader("Time Step Test"); + FastOS_Time before, timing; + before.SetNow(); + + int delay = 400; + for(int i=delay; i<3000; i+=delay) + { + FastOS_Thread::Sleep(delay); + + timing.SetNow(); + timing -= before; + + double millis = timing.MilliSecs(); + double correct = i; + + Progress((fabs(millis - correct)/correct) < 0.15, + "Elapsed time measurement: %d", + static_cast<int>(millis)); + } + + PrintSeparator(); + } + + void requireThatTimeStampCanBeConvertedToString() { + TestHeader("requireThatTimeStampCanBeConvertedToString"); + + int64_t time = 1424867106 * fastos::TimeStamp::SEC + 123 * fastos::TimeStamp::MS; + fastos::TimeStamp timeStamp(time); + std::string actualString = timeStamp.toString(); + std::string expectString = "2015-02-25 12:25:06.123 UTC"; + Progress(expectString == actualString, + "Actual string: '%s'", actualString.c_str()); + + PrintSeparator(); + } + + void requireThatTimeStampIsConstructedCorrect() { + using fastos::TimeStamp; + Progress(TimeStamp(97).ns() == 97l, "TimeStamp(int)"); + Progress(TimeStamp(97u).ns() == 97l, "TimeStamp(unsigned int)"); + Progress(TimeStamp(97l).ns() == 97l, "TimeStamp(long)"); + Progress(TimeStamp(97ul).ns() == 97l, "TimeStamp(unsigned long)"); + Progress(TimeStamp(TimeStamp::Seconds(97.3)).ns() == 97300000000l, "TimeStamp(double)"); + PrintSeparator(); + } + + + int Main (); +}; + +int TimeTest::Main () +{ + printf("grep for the string '%s' to detect failures.\n\n", failString); + + Wait3SecondsTest(); + TimeArithmeticTest(); + TimeStepTest(); + requireThatTimeStampCanBeConvertedToString(); + requireThatTimeStampIsConstructedCorrect(); + + printf("END OF TEST (%s)\n", _argv[0]); + + return 0; +} + + +int main (int argc, char **argv) +{ + TimeTest app; + + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} + diff --git a/fastos/src/tests/typetest.cpp b/fastos/src/tests/typetest.cpp new file mode 100644 index 00000000000..5439c1ad071 --- /dev/null +++ b/fastos/src/tests/typetest.cpp @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> + +#include <vespa/fastos/fastos.h> +#include "tests.h" + + +class TypeTest : public BaseTest +{ +private: + void PrintfSpecifiersTest () + { + char testBuf[200]; + const char *correct; + + TestHeader("64-bit printf-specifiers test"); + + sprintf(testBuf, "%" PRId64, int64_t(-1000)*1000*1000*1000*1000); + correct = "-1000000000000000"; + Progress(strcmp(testBuf, correct) == 0, + "Generated=[%s], Correct=[%s]", testBuf, correct); + + sprintf(testBuf, "%" PRIu64, uint64_t(1000)*1000*1000*1000*1000); + correct = "1000000000000000"; + Progress(strcmp(testBuf, correct) == 0, + "Generated=[%s], Correct=[%s]", testBuf, correct); + + sprintf(testBuf, "%" PRIo64, uint64_t(1000)*1000*1000*1000*1000); + correct = "34327724461500000"; + Progress(strcmp(testBuf, correct) == 0, + "Generated=[%s], Correct=[%s]", testBuf, correct); + + sprintf(testBuf, "%" PRIx64, uint64_t(1000)*1000*1000*1000*1000); + correct = "38d7ea4c68000"; + Progress(strcmp(testBuf, correct) == 0, + "Generated=[%s], Correct=[%s]", testBuf, correct); + + sprintf(testBuf, "%" PRIX64, uint64_t(1000)*1000*1000*1000*1000); + correct = "38D7EA4C68000"; + Progress(strcmp(testBuf, correct) == 0, + "Generated=[%s], Correct=[%s]", testBuf, correct); + + PrintSeparator(); + } + + void ObjectSizeTest () + { + TestHeader("Object Sizes (bytes)"); + + Progress(true, "FastOS_Application: %d", sizeof(FastOS_Application)); + Progress(true, "FastOS_BoolCond %d", sizeof(FastOS_BoolCond)); + Progress(true, "FastOS_Cond %d", sizeof(FastOS_Cond)); + Progress(true, "FastOS_DirectoryScan %d", sizeof(FastOS_DirectoryScan)); + Progress(true, "FastOS_File: %d", sizeof(FastOS_File)); + Progress(true, "FastOS_Mutex: %d", sizeof(FastOS_Mutex)); + Progress(true, "FastOS_Runnable %d", sizeof(FastOS_Runnable)); + Progress(true, "FastOS_ServerSocket %d", sizeof(FastOS_ServerSocket)); + Progress(true, "FastOS_Socket: %d", sizeof(FastOS_Socket)); + Progress(true, "FastOS_SocketFactory %d", sizeof(FastOS_SocketFactory)); + Progress(true, "FastOS_StatInfo %d", sizeof(FastOS_StatInfo)); + Progress(true, "FastOS_Thread: %d", sizeof(FastOS_Thread)); + Progress(true, "FastOS_ThreadPool: %d", sizeof(FastOS_ThreadPool)); + Progress(true, "FastOS_Time %d", sizeof(FastOS_Time)); + + PrintSeparator(); + } + +public: + virtual ~TypeTest() {}; + + int Main () + { + printf("grep for the string '%s' to detect failures.\n\n", failString); + + PrintfSpecifiersTest(); + ObjectSizeTest(); + + PrintSeparator(); + printf("END OF TEST (%s)\n", _argv[0]); + + return 0; + } +}; + + +int main (int argc, char **argv) +{ + setvbuf(stdout, NULL, _IOLBF, 8192); + TypeTest app; + return app.Entry(argc, argv); +} + diff --git a/fastos/src/tests/usecputest.cpp b/fastos/src/tests/usecputest.cpp new file mode 100644 index 00000000000..3ddc46aaa7e --- /dev/null +++ b/fastos/src/tests/usecputest.cpp @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "tests.h" + +class ThreadRunJob; +void UseSomeCpu(int i, ThreadRunJob *threadRunJob); + +class ThreadRunJob : public FastOS_Runnable +{ +public: + int64_t UseSomeCpu2(int64_t someNumber) + { + return someNumber + (someNumber/2 + someNumber*4) + + someNumber * someNumber * someNumber; + } + + void Run (FastOS_ThreadInterface *thisThread, void *arg) + { + (void)thisThread; + (void)arg; + + FastOS_Time before, current; + before.SetNow(); + + for(int i=0; i<200000; i++) + { + if((i % 200) == 0) + { + current.SetNow(); + current -= before; + if(current.MilliSecs() > 3000) + break; + } + UseSomeCpu(i, this); + } + delete (this); + } +}; + +class UseCpuTest : public BaseTest +{ +public: + int Main () + { + FastOS_ThreadPool pool(128*1024); + pool.NewThread(new ThreadRunJob()); + pool.NewThread(new ThreadRunJob()); + pool.NewThread(new ThreadRunJob()); + pool.NewThread(new ThreadRunJob()); + pool.Close(); + return 0; + } +}; + +void UseSomeCpu (int i, ThreadRunJob *threadRunJob) +{ + int64_t lastVal = i; + for(int e=0; e<100; e++) + lastVal = threadRunJob->UseSomeCpu2(lastVal); +} + +int main (int argc, char **argv) +{ + UseCpuTest app; + setvbuf(stdout, NULL, _IOLBF, 8192); + return app.Entry(argc, argv); +} + diff --git a/fastos/src/vespa/fastos/.gitignore b/fastos/src/vespa/fastos/.gitignore new file mode 100644 index 00000000000..004799df5b4 --- /dev/null +++ b/fastos/src/vespa/fastos/.gitignore @@ -0,0 +1,28 @@ +*.So +*.core +*.exe +*.ilk +*.pdb +.depend +.depend.NEW +Debug +Makefile +Makefile.factory +Makefile.overrides +Makefile.pre +Release +autoconf.h +config_command.bat +config_command.sh +fastconfig +fastconfig.exe +fastos.lib +fastosconfig.h +libfastos.a +makefeatures +makemake +processtest.log +test.txt +vc60.idb +vc60.pdb +/libfastos.so.5.1 diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt new file mode 100644 index 00000000000..782c704624f --- /dev/null +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(fastos + SOURCES + app.cpp + backtrace.c + file.cpp + linux_file.cpp + serversocket.cpp + socket.cpp + socketevent.cpp + thread.cpp + time.cpp + timestamp.cpp + unix_app.cpp + unix_cond.cpp + unix_dynamiclibrary.cpp + unix_file.cpp + unix_ipc.cpp + unix_mutex.cpp + unix_process.cpp + unix_socket.cpp + unix_thread.cpp + unix_time.cpp + vtag.cpp + INSTALL lib64 + DEPENDS +) +find_package(Threads REQUIRED) +target_link_libraries(fastos PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fastos/src/vespa/fastos/app.cpp b/fastos/src/vespa/fastos/app.cpp new file mode 100644 index 00000000000..282f1250202 --- /dev/null +++ b/fastos/src/vespa/fastos/app.cpp @@ -0,0 +1,176 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Implementation of FastOS_ApplicationInterface methods. + * + * @author Div, Oivind H. Danielsen + */ + +#include <vespa/fastos/app.h> +#include <vespa/fastos/socket.h> +#include <vespa/fastos/file.h> + +#include <vespa/fastos/process.h> +#include <vespa/fastos/thread.h> + +FastOS_ApplicationInterface *FastOS_ProcessInterface::_app; +FastOS_ThreadPool *FastOS_ProcessInterface::GetThreadPool () +{ + return _app->GetThreadPool(); +} + +FastOS_ThreadPool *FastOS_ApplicationInterface::GetThreadPool () +{ + return _threadPool; +} + +FastOS_ApplicationInterface::FastOS_ApplicationInterface() : + _threadPool(NULL), + _processList(NULL), + _processListMutex(NULL), + _disableLeakReporting(false), + _argc(0), + _argv(NULL) +{ + FastOS_ProcessInterface::_app = this; + char * fadvise = getenv("VESPA_FADVISE_OPTIONS"); + if (fadvise != NULL) { + int fadviseOptions(0); + if (strstr(fadvise, "SEQUENTIAL")) { fadviseOptions |= POSIX_FADV_SEQUENTIAL; } + if (strstr(fadvise, "RANDOM")) { fadviseOptions |= POSIX_FADV_RANDOM; } + if (strstr(fadvise, "WILLNEED")) { fadviseOptions |= POSIX_FADV_WILLNEED; } + if (strstr(fadvise, "DONTNEED")) { fadviseOptions |= POSIX_FADV_DONTNEED; } + if (strstr(fadvise, "NOREUSE")) { fadviseOptions |= POSIX_FADV_NOREUSE; } + FastOS_FileInterface::setDefaultFAdviseOptions(fadviseOptions); + } +} + +FastOS_ApplicationInterface::~FastOS_ApplicationInterface () +{ +} + +bool FastOS_ApplicationInterface::Init () +{ + bool rc=false; + + if(PreThreadInit()) + { + if(FastOS_Thread::InitializeClass()) + { + if(FastOS_File::InitializeClass()) + { + const char *errorMsg = FastOS_Socket::InitializeServices(); + + if(errorMsg == NULL) + { + _processListMutex = new FastOS_Mutex(); + _threadPool = new FastOS_ThreadPool(128 * 1024); + rc = true; + } + else + { + fprintf(stderr, + "FastOS_Socket::InitializeServices() returned:\n[%s]\n", + errorMsg); + } + } + else + fprintf(stderr, "FastOS_File class initialization failed.\n"); + } + else + fprintf(stderr, "FastOS_Thread class initialization failed.\n"); + } + else + fprintf(stderr, "FastOS_PreThreadInit failed.\n"); + + return rc; +} + + +void FastOS_ApplicationInterface::Cleanup () +{ + if(_threadPool != NULL) + { + // printf("Closing threadpool...\n"); + _threadPool->Close(); + // printf("Deleting threadpool...\n"); + delete _threadPool; + _threadPool = NULL; + } + + if(_processListMutex != NULL) + { + delete _processListMutex; + _processListMutex = NULL; + } + + FastOS_Socket::CleanupServices(); + FastOS_File::CleanupClass(); + FastOS_Thread::CleanupClass(); +} + +int FastOS_ApplicationInterface::Entry (int argc, char **argv) +{ + int rc=255; + + _argc = argc; + _argv = argv; + + if(Init()) + { + rc = Main(); + } + + Cleanup(); + + return rc; +} + +void FastOS_ApplicationInterface:: +OnReceivedIPCMessage (const void *data, size_t length) +{ + (void)data; + (void)length; + // Default dummy handler +} + +void FastOS_ApplicationInterface:: +AddChildProcess (FastOS_ProcessInterface *node) +{ + node->_prev = NULL; + node->_next = _processList; + + if(_processList != NULL) + _processList->_prev = node; + + _processList = node; +} + +void FastOS_ApplicationInterface:: +RemoveChildProcess (FastOS_ProcessInterface *node) +{ + if(node->_prev) + node->_prev->_next = node->_next; + else + _processList = node->_next; + + if(node->_next) + { + node->_next->_prev = node->_prev; + node->_next = NULL; + } + + if(node->_prev != NULL) + node->_prev = NULL; +} + +bool +FastOS_ApplicationInterface::useProcessStarter() const +{ + return false; +} +bool +FastOS_ApplicationInterface::useIPCHelper() const +{ + return useProcessStarter(); +} diff --git a/fastos/src/vespa/fastos/app.h b/fastos/src/vespa/fastos/app.h new file mode 100644 index 00000000000..b025efa9ea2 --- /dev/null +++ b/fastos/src/vespa/fastos/app.h @@ -0,0 +1,286 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definition for FastOS_ApplicationInterface. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + + +#include <vespa/fastos/types.h> + +class FastOS_ProcessInterface; + +#include <vespa/fastos/thread.h> +#include <vespa/fastos/mutex.h> + +/** + * FastOS application wrapper class. + * This class manages initialization and cleanup of the services + * provided by FastOS. + * Your application should inherit from this class, and implement + * @ref Main(). + * It is possible, but not required to override the @ref Init() + * and @ref Cleanup() methods. If you do, make sure you invoke + * the superclass, in order to perform the neccessary initialization + * and cleanup. + * + * To startup the FastOS_Application, invoke @ref Entry() as shown + * in the example below. This will call @ref Init(), @ref Main() + * (if @ref Init() succeeds) and @ref Cleanup(), in that order. + * + * Example: + * @code + * #include <vespa/fastos/fastos.h> + * + * class MyApplication : public FastOS_Application + * { + * int Main () + * { + * printf("Hello world\n"); + * + * return 0; + * } + * }; + * + * + * int main (int argc, char **argv) + * { + * MyApplication app; + * + * app.Entry(argc, argv); + * } + * @endcode + * + * Here is another example. This version overrides @ref Init() and + * @ref Cleanup(). Most applications do not need to do this, but + * an example of how to do it is included anyway: + * @code + * #include <vespa/fastos/fastos.h> + * + * class MyApplication : public FastOS_Application + * { + * bool MyOwnInitializationStuff () + * { + * printf("My initialization stuff\n"); + * return true; + * } + * + * void MyOwnCleanupStuff() + * { + * printf("My cleanup stuff\n"); + * } + * + * // Most applications do not need to override this method. + * // If you do, make sure you invoke the superclass Init() + * // first. + * bool Init () + * { + * bool rc=false; + * + * if(FastOS_Application::Init()) + * { + * if(MyOwnInitializationStuff()) + * { + * rc = true; + * } + * } + * + * // The application will not start if false is returned. + * return rc; + * } + * + * // Most applications do not need to override this method. + * // If you do, make sure you invoke the superclass Cleanup() + * // at the end. + * // Note that Cleanup() is always invoked, even when Init() + * // fails. + * void Cleanup () + * { + * MyOwnCleanupStuff(); + * + * FastOS_Application::Cleanup(); + * } + * + * int Main () + * { + * printf("Hello world\n"); + * + * return 0; + * } + * }; + * + * + * int main (int argc, char **argv) + * { + * MyApplication app; + * + * app.Entry(argc, argv); + * } + * + * @endcode + */ +class FastOS_ApplicationInterface +{ + friend int main (int argc, char **argv); + +private: + FastOS_ApplicationInterface(const FastOS_ApplicationInterface&); + FastOS_ApplicationInterface& operator=(const FastOS_ApplicationInterface&); + +protected: + /** + * + * Indicate if a process starter is going to be used. + * Only override this one if you are going to start other processes. + * @return true if you are going to use a process starter. + */ + virtual bool useProcessStarter() const; + virtual bool useIPCHelper() const; + + FastOS_ThreadPool *_threadPool; + FastOS_ProcessInterface *_processList; + FastOS_Mutex *_processListMutex; + + bool _disableLeakReporting; + virtual bool PreThreadInit () { return true; } + +public: + int _argc; + char **_argv; + + FastOS_ApplicationInterface(); + + virtual ~FastOS_ApplicationInterface(); + + /** + * FastOS initialization. + * This method performs the neccessary initialization for FastOS. + * It is possible to override this method, see the class documentation + * for an example. + * @ref Main() will be called, if and only if @ref Init() returns true. + * @ref Cleanup will always be called, regardless of the @ref Init() + * return value. + */ + virtual bool Init(); + + /** + * FastOS Entry. + * This method is used to enter the application and provide command + * line arguments. See the class documentation for an example on + * how this is used. + * @param argc Number of arguments + * @param argv Array of arguments + * @return Errorlevel returned to the shell + */ + int Entry (int argc, char **argv); + + /** + * FastOS Main. + * This is where your application starts. See the class documentation + * for an example on how this is used. + * @return Errorlevel returned to the shell + */ + virtual int Main ()=0; + + /** + * FastOS Cleanup. + * This method performs the neccessary cleanup for FastOS. + * It is possible to override this method, see the class documentation + * for an example. + * @ref Cleanup() is always called, regardless of the return values + * of @ref Init() and @ref Main(). + */ + virtual void Cleanup (); + + /** + * Parse program arguments. @ref GetOpt() incrementally parses the + * command line argument list and returns the next known option + * character. An option character is known if it has been + * specified in the string of accepted option characters, + * [optionsString]. + * + * The option string [optionsString] may contain the following + * elements: individual characters, and characters followed by a + * colon to indicate an option argument is to follow. For example, + * an option string "x" recognizes an option ``-x'', and an option + * string "x:" recognizes an option and argument ``-x argument''. + * It does not matter to @ref GetOpt() if a following argument has + * leading white space. + * + * @ref GetOpt() returns -1 when the argument list is exhausted, or + * `?' if a non-recognized option is encountered. The + * interpretation of options in the argument list may be canceled + * by the option `--' (double dash) which causes getopt() to signal + * the end of argument processing and return -1. When all options + * have been processed (i.e., up to the first non-option argument), + * getopt() returns -1. + * + * @ref GetOpt() should only be run by a single thread at the same + * time. In order to evaluate the argument list multiple times, the + * previous GetOpt loop must be finished (-1 returned). + */ + char GetOpt (const char *optionsString, + const char* &optionArgument, + int &optionIndex); + + /** + * This method is invoked each time an IPC message is received. + * The default implementation discards the message. Subclass this + * method to process the data. You should assume that any + * thread can invoke this method. + * @param data Pointer to binary message data + * @param length Length of message in bytes + */ + virtual void OnReceivedIPCMessage (const void *data, size_t length); + + /** + * Send an IPC message to the parent process. The method fails + * if the parent process is not a FastOS process. + * @param data Pointer to binary message data + * @param length Length of message in bytes + * @return Boolean success/failure + */ + virtual bool SendParentIPCMessage (const void *data, size_t length) = 0; + + void AddChildProcess (FastOS_ProcessInterface *node); + void RemoveChildProcess (FastOS_ProcessInterface *node); + void ProcessLock () { _processListMutex->Lock(); } + void ProcessUnlock() { _processListMutex->Unlock(); } + FastOS_ProcessInterface *GetProcessList () { return _processList; } + + FastOS_ThreadPool *GetThreadPool (); + + /** + * Disable reporting of memory- and other resource leaks. + * If you want to disable leak reporting, call this method + * before FastOS_Application::Entry() is invoked, as irreversible + * actions which set up leak reporting can be performed at this point. + * Leak reporting is either performed during FastOS_Application::Cleanup() + * or when the application process terminates. + * Leak reporting is currently only supported with the debug build + * on Win32. + */ + void DisableLeakReporting () { _disableLeakReporting = true; } +}; + + +#include <vespa/fastos/unix_app.h> +typedef FastOS_UNIX_Application FASTOS_PREFIX(Application); + + + +// Macro included for convenience. +#define FASTOS_MAIN(application_class) \ +int main (int argc, char **argv) \ +{ \ + application_class app; \ + \ + return app.Entry(argc, argv); \ +} + + + diff --git a/fastos/src/vespa/fastos/backtrace.c b/fastos/src/vespa/fastos/backtrace.c new file mode 100644 index 00000000000..b20f20d1fc0 --- /dev/null +++ b/fastos/src/vespa/fastos/backtrace.c @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "backtrace.h" + +#if defined(__i386__) || defined(__clang__) +// use GLIBC version, hope it works +extern int backtrace(void **buffer, int size); +#define HAVE_BACKTRACE +#endif + +#if defined(__x86_64__) && !defined(__clang__) + +/** + Written by Arne H. J. based on docs: + + http://www.kernel.org/pub/linux/devel/gcc/unwind/ + http://www.codesourcery.com/public/cxx-abi/abi-eh.html + http://refspecs.freestandards.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/libgcc-s-ddefs.html +**/ + +#include <unwind.h> + +struct trace_context { + void **array; + int size; + int index; +}; + +static _Unwind_Reason_Code +trace_fn(struct _Unwind_Context *ctxt, void *arg) +{ + struct trace_context *tp = (struct trace_context *)arg; + void *ip = (void *)_Unwind_GetIP(ctxt); + + if (ip == 0) { + return _URC_END_OF_STACK; + } + if (tp->index <= tp->size) { + // there's no point filling in the address of the backtrace() + // function itself, that doesn't provide any extra information, + // so skip one level + if (tp->index > 0) { + tp->array[tp->index - 1] = ip; + } + tp->index++; + } else { + return _URC_NORMAL_STOP; + } + return _URC_NO_REASON; // "This is not the destination frame" -> try next frame +} + +#define HAVE_BACKTRACE +int +backtrace (void **array, int size) +{ + struct trace_context t; + t.array = array; + t.size = size; + t.index = 0; + _Unwind_Backtrace(trace_fn, &t); + return t.index - 1; +} +#endif // x86_64 + + +#ifdef HAVE_BACKTRACE + +int +FastOS_backtrace (void **array, int size) +{ + return backtrace(array, size); +} + +#else + +# warning "backtrace not supported on this CPU" +int +FastOS_backtrace (void **array, int size) +{ + (void) array; + (void) size; + return 0; +} + +#endif diff --git a/fastos/src/vespa/fastos/backtrace.h b/fastos/src/vespa/fastos/backtrace.h new file mode 100644 index 00000000000..45c1ef1378d --- /dev/null +++ b/fastos/src/vespa/fastos/backtrace.h @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +int FastOS_backtrace (void **array, int size); + +#if defined(__x86_64__) +int backtrace (void **array, int size); +#endif + +#ifdef __cplusplus +} +#endif + diff --git a/fastos/src/vespa/fastos/cond.h b/fastos/src/vespa/fastos/cond.h new file mode 100644 index 00000000000..1f8f4f12522 --- /dev/null +++ b/fastos/src/vespa/fastos/cond.h @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definitions for FastOS_CondInterface and FastOS_BoolCond. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + + +#include <vespa/fastos/types.h> +#include <vespa/fastos/mutex.h> + + +/** + * This class implements a synchronization mechanism used by threads to wait + * until a condition expression involving shared data attains a particular state. + * + * Condition variables provide a different type of synchronization + * than locking mechanisms like mutexes. For instance, a mutex is used + * to cause other threads to wait while the thread holding the mutex + * executes code in a critical section. In contrast, a condition + * variable is typically used by a thread to make itself wait until an + * expression involving shared data attains a particular state. + */ +class FastOS_CondInterface : public FastOS_Mutex +{ +public: + FastOS_CondInterface(void) : FastOS_Mutex() { } + + virtual ~FastOS_CondInterface () {} + + /** + * Wait for the condition to be signalled. If the wait takes + * longer than [milliseconds] ms, the wait is aborted and false + * is returned. + * @param milliseconds Max time to wait. + * @return Boolean success/failure + */ + virtual bool TimedWait (int milliseconds) = 0; + + /** + * Wait for the condition to be signalled. + */ + virtual void Wait (void)=0; + + /** + * Send a signal to one thread waiting on the condition (if any). + */ + virtual void Signal (void)=0; + + /** + * Send a signal to all threads waiting on the condition. + */ + virtual void Broadcast (void)=0; +}; + +#include <vespa/fastos/unix_cond.h> +typedef FastOS_UNIX_Cond FASTOS_PREFIX(Cond); + +/** + * This class implements a condition variable with a boolean + * value. + */ +class FastOS_BoolCond : public FastOS_Cond +{ + bool _busy; + +public: + /** + * Constructor. Initially the boolean variable is + * set to non-busy. + */ + FastOS_BoolCond(void) : _busy(false) { } + + ~FastOS_BoolCond(void) { } + + /** + * If the variable is busy, wait for it to be non-busy, + * then set the variable to busy. */ + void SetBusy(void) + { + Lock(); + + while (_busy == true) + Wait(); + + _busy = true; + Unlock(); + } + + /** + * If the variable is busy, wait until it is no longer busy. + * If it was non-busy to begin with, no wait is performed. + */ + void WaitBusy(void) + { + Lock(); + + while (_busy == true) + Wait(); + + Unlock(); + } + + /** + * If the variable is busy, wait until it is no longer busy or a + * timeout occurs. If it was non-busy to begin with, no wait is + * performed. + * @param ms Time to wait + * @return True=non-busy, false=timeout + */ + bool TimedWaitBusy(int ms) + { + bool success = true; + + Lock(); + if (_busy == true) { + success = TimedWait(ms); + } + Unlock(); + + return success; + } + + /** + * Return busy status. + * @return True=busy, false=non-busy + */ + bool PollBusy (void) + { + bool rc; + Lock(); + rc = _busy; + Unlock(); + return rc; + } + + /** + * Set the variable to non-busy, and signal one thread + * waiting (if there are any). + * (if any). + */ + void ClearBusy(void) + { + Lock(); + _busy = false; + Signal(); + Unlock(); + } + + /** + * Set the variable to non-busy, and broadcast to all + * threads waiting (if there are any). + */ + void ClearBusyBroadcast(void) + { + Lock(); + _busy = false; + Broadcast(); + Unlock(); + } +}; + + diff --git a/fastos/src/vespa/fastos/dynamiclibrary.h b/fastos/src/vespa/fastos/dynamiclibrary.h new file mode 100644 index 00000000000..c4154a612f1 --- /dev/null +++ b/fastos/src/vespa/fastos/dynamiclibrary.h @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/****************************************************************-*-C++-*- + * @file + * Class definitions for FastOS_DynamicLibrary. + * + * @author Eyvind Bernhardsen + * + * Creation date : 2003-07-02 + *************************************************************************/ + + + +#pragma once + + +#include <vespa/fastos/types.h> +#include <string> + +/** + * This class contains functionality to load, get symbols from and + * unload dynamic libraries. + */ + +class FastOS_DynamicLibraryInterface +{ +public: + /** + * Destructor. The destructor will close the library if it is open. + */ + virtual ~FastOS_DynamicLibraryInterface() {} + + /** + * Open (load) the library. + * @param libname the name of the library to open + * @return Boolean success/failure + */ + virtual bool Open(const char *libname = NULL) = 0; + + /** + * Close (unload) the library. + * @return Boolean success/failure + */ + virtual bool Close() = 0; + + /** + * Find the address of a symbol in the library. + * @param symbol Name of symbol to find + * @return Address of the symbol, or NULL if an error has occurred + */ + virtual void * GetSymbol(const char *symbol) const = 0; + + /** + * Check if the library is open. + * @return true if it is, false if it ain't + */ + virtual bool IsOpen() const = 0; + + /** + * Return an error message describing the last error. This is + * currently platform-dependent, unfortunately; FastOS does not + * normalize the error messages. + * @return The error string if an error has occurred since the last + * invocation, or an empty one if no error has occurred. + */ + std::string GetLastErrorString(); +}; + + +# include "unix_dynamiclibrary.h" +typedef FastOS_UNIX_DynamicLibrary FASTOS_PREFIX(DynamicLibrary); + +/********************************************************************* + * Dynamic library helper macros: + * + * FASTOS_LOADABLE_EXPORT prefix that marks a symbol to be exported + * FASTOS_LOADABLE_IMPORT prefix that marks a symbol to be imported + * from a dll + * FASTOS_LOADABLE_FACTORY macro that creates and exports a function + * called factory. The macro takes a class + * name as its only parameter, and the + * factory function returns a pointer to an + * instance of that class. + * + * Example usage: + * loadableclass.h: + * class FastOS_LoadableClass + * { + * public: + * void DoSomething(); + * } + * + * in loadableclass.cpp: + * FASTOS_LOADABLE_FACTORY(LoadableClass) + *********************************************************************/ + + +# define FASTOS_LOADABLE_EXPORT + +# define FASTOS_LOADABLE_IMPORT + +#define FASTOS_LOADABLE_FACTORY(loadable_class) \ +extern "C" { \ + FASTOS_LOADABLE_EXPORT loadable_class *factory() { \ + return new loadable_class; \ + } \ +} + +// New macros to support the new gcc visibility features. + +#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/fastos/src/vespa/fastos/fastos.h b/fastos/src/vespa/fastos/fastos.h new file mode 100644 index 00000000000..dd03a3c512c --- /dev/null +++ b/fastos/src/vespa/fastos/fastos.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Main include file for FastOS. This pulls in all the neccessary + * definitions. + * + * @author Div, Oivind H. Danielsen + */ + + +#pragma once + +#include <vespa/fastos/types.h> +#include <vespa/fastos/app.h> +#include <vespa/fastos/file.h> +#include <vespa/fastos/mutex.h> +#include <vespa/fastos/cond.h> +#include <vespa/fastos/thread.h> +#include <vespa/fastos/socket.h> +#include <vespa/fastos/serversocket.h> +#include <vespa/fastos/timestamp.h> +#include <vespa/fastos/time.h> +#include <vespa/fastos/prefetch.h> +#include <vespa/fastos/process.h> +#include <vespa/fastos/dynamiclibrary.h> + diff --git a/fastos/src/vespa/fastos/file.cpp b/fastos/src/vespa/fastos/file.cpp new file mode 100644 index 00000000000..2b3052d9e87 --- /dev/null +++ b/fastos/src/vespa/fastos/file.cpp @@ -0,0 +1,516 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Implementation of FastOS_FileInterface methods. + * + * @author Div, Oivind H. Danielsen + */ + +#include <vespa/fastos/file.h> +#include <sstream> +#include <stdexcept> + + +DirectIOException::DirectIOException(const char * fileName, const void * buffer, size_t length, int64_t offset) : + std::exception(), + _what(), + _fileName(fileName), + _buffer(buffer), + _length(length), + _offset(offset) +{ + std::ostringstream os; + os << "DirectIO failed for file '" << fileName << "' buffer=0x" << std::hex << reinterpret_cast<size_t>(buffer); + os << " length=0x" << length << " offset=0x" << offset; + _what = os.str(); +} + +FastOS_FileInterface::FailedHandler FastOS_FileInterface::_failedHandler = NULL; +int FastOS_FileInterface::_defaultFAdviseOptions = POSIX_FADV_NORMAL; + +static const size_t MAX_WRITE_CHUNK_SIZE = 0x4000000; // 64 MB + +FastOS_FileInterface::FastOS_FileInterface(const char *filename) + : _fAdviseOptions(_defaultFAdviseOptions), + _writeChunkSize(MAX_WRITE_CHUNK_SIZE), + _filename(NULL), + _openFlags(0), + _directIOEnabled(false), + _syncWritesEnabled(false) +{ + if (filename != NULL) + SetFileName(filename); +} + + +FastOS_FileInterface::~FastOS_FileInterface(void) +{ + free(_filename); +} + +bool FastOS_FileInterface::InitializeClass (void) +{ + return true; +} + +bool FastOS_FileInterface::CleanupClass (void) +{ + return true; +} + +void +FastOS_FileInterface::ReadBuf(void *buffer, size_t length) +{ + ssize_t readResult = Read(buffer, length); + + if ((readResult == -1) || (static_cast<size_t>(readResult) != length)) { + std::string errorString = readResult != -1 ? + std::string("short read") : + FastOS_FileInterface::getLastErrorString(); + std::ostringstream os; + os << "Fatal: Reading " << length << " bytes from '" << GetFileName() << "' failed: " << errorString; + throw std::runtime_error(os.str()); + } +} + +void +FastOS_FileInterface::WriteBuf(const void *buffer, size_t length) +{ + const char * data = static_cast<const char *>(buffer); + while (length > 0) { + size_t len = std::min(_writeChunkSize, length); + WriteBufInternal(data, len); + data += len; + length -= len; + } +} + +void +FastOS_FileInterface::WriteBufInternal(const void *buffer, size_t length) +{ + ssize_t writeResult = Write2(buffer, length); + if (length - writeResult != 0) { + std::string errorString = writeResult != -1 ? + std::string("short write") : + FastOS_FileInterface::getLastErrorString(); + std::ostringstream os; + os << "Fatal: Writing " << length << " bytes to '" << GetFileName() << "' failed (wrote " << writeResult << "): " << errorString; + throw std::runtime_error(os.str()); + } +} + +bool +FastOS_FileInterface::CheckedWrite(const void *buffer, size_t len) +{ + ssize_t writeResult = Write2(buffer, len); + if (writeResult < 0) { + std::string errorString = FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, "Writing %lu bytes to '%s' failed: %s\n", + static_cast<unsigned long>(len), + GetFileName(), + errorString.c_str()); + return false; + } + if (writeResult != (ssize_t)len) { + fprintf(stderr, "Short write, tried to write %lu bytes to '%s', only wrote %lu bytes\n", + static_cast<unsigned long>(len), + GetFileName(), + static_cast<unsigned long>(writeResult)); + return false; + } + return true; +} + + +void +FastOS_FileInterface::ReadBuf(void *buffer, size_t length, int64_t readOffset) +{ + if (!SetPosition(readOffset)) { + std::string errorString = FastOS_FileInterface::getLastErrorString(); + std::ostringstream os; + os << "Fatal: Setting fileoffset to " << readOffset << " in '" << GetFileName() << "' : " << errorString; + throw std::runtime_error(os.str()); + } + ReadBuf(buffer, length); +} + + +void +FastOS_FileInterface::EnableDirectIO(void) +{ + // Only subclasses with support for DirectIO do something here. +} + + +void +FastOS_FileInterface::SetWriteChunkSize(size_t writeChunkSize) +{ + _writeChunkSize = writeChunkSize; +} + + +void +FastOS_FileInterface::EnableSyncWrites(void) +{ + if (!IsOpened()) + _syncWritesEnabled = true; +} + + +bool +FastOS_FileInterface:: +GetDirectIORestrictions(size_t &memoryAlignment, + size_t &transferGranularity, + size_t &transferMaximum) +{ + memoryAlignment = 1; + transferGranularity = 1; + transferMaximum = 0x7FFFFFFF; + return false; +} + +bool +FastOS_FileInterface::DirectIOPadding(int64_t offset, + size_t buflen, + size_t &padBefore, + size_t &padAfter) +{ + (void)offset; + (void)buflen; + padBefore = 0; + padAfter = 0; + return false; +} + + +void * +FastOS_FileInterface::AllocateDirectIOBuffer(size_t byteSize, void *&realPtr) +{ + realPtr = malloc(byteSize); // Default - use malloc allignment + return realPtr; +} + + +void +FastOS_FileInterface::enableMemoryMap(int mmapFlags) +{ + // Only subclases with support for memory mapping do something here. + (void) mmapFlags; +} + + +void * +FastOS_FileInterface::MemoryMapPtr(int64_t position) const +{ + // Only subclases with support for memory mapping do something here. + (void) position; + return NULL; +} + + +bool +FastOS_FileInterface::IsMemoryMapped(void) const +{ + // Only subclases with support for memory mapping do something here. + return false; +} + +bool +FastOS_FileInterface::CopyFile( const char *src, const char *dst ) +{ + FastOS_File s, d; + FastOS_StatInfo statInfo; + bool success = false; + + if ( src != NULL && + dst != NULL && + strcmp(src, dst) != 0 && + FastOS_File::Stat( src, &statInfo )) { + + if ( s.OpenReadOnly( src ) && d.OpenWriteOnlyTruncate( dst ) ) { + + unsigned int bufSize = 1024*1024; + int64_t bufSizeBound = statInfo._size; + if (bufSizeBound < 1) + bufSizeBound = 1; + if (bufSizeBound < static_cast<int64_t>(bufSize)) + bufSize = static_cast<unsigned int>(bufSizeBound); + char *tmpBuf = new char[ bufSize ]; + + if ( tmpBuf != NULL ) { + int64_t copied = 0; + success = true; + do { + unsigned int readBytes = s.Read( tmpBuf, bufSize ); + if (readBytes > 0) { + + if ( !d.CheckedWrite( tmpBuf, readBytes)) { + success = false; + } + copied += readBytes; + } else { + // Could not read from src. + success = false; + } + } while (copied < statInfo._size && success); + + delete [] tmpBuf; + } // else out of memory ? + + s.Close(); + d.Close(); + } // else Could not open source or destination file. + } // else Source file does not exist, or input args are invalid. + + return success; +} + + +bool +FastOS_FileInterface::MoveFile(const char* src, const char* dst) +{ + bool rc = FastOS_File::Rename(src, dst); + if (!rc) { + // Try copy and remove. + if (CopyFile(src, dst)) { + rc = FastOS_File::Delete(src); + } + } + return rc; +} + + +void +FastOS_FileInterface::EmptyDirectory( const char *dir, + const char *keepFile /* = NULL */ ) +{ + FastOS_StatInfo statInfo; + if (!FastOS_File::Stat(dir, &statInfo)) + return; // Fail if the directory does not exist + FastOS_DirectoryScan dirScan( dir ); + + while (dirScan.ReadNext()) { + if (strcmp(dirScan.GetName(), ".") != 0 && + strcmp(dirScan.GetName(), "..") != 0 && + (keepFile == NULL || strcmp(dirScan.GetName(), keepFile) != 0)) + { + std::string name = dir; + name += GetPathSeparator(); + name += dirScan.GetName(); + if (dirScan.IsDirectory()) { + EmptyAndRemoveDirectory(name.c_str()); + } else { + if ( ! FastOS_File::Delete(name.c_str()) ) { + std::ostringstream os; + os << "Failed deleting file '" << name << "' due to " << getLastErrorString(); + throw std::runtime_error(os.str().c_str()); + } + } + } + } +} + + +void +FastOS_FileInterface::EmptyAndRemoveDirectory(const char *dir) +{ + EmptyDirectory(dir); + FastOS_File::RemoveDirectory(dir); +} + +void +FastOS_FileInterface::MakeDirIfNotPresentOrExit(const char *name) +{ + FastOS_StatInfo statInfo; + + if (FastOS_File::Stat(name, &statInfo)) { + if (statInfo._isDirectory) + return; + + fprintf(stderr, "%s is not a directory\n", name); + exit(1); + } + + if (statInfo._error != FastOS_StatInfo::FileNotFound) { + char errorBuf[100]; + int error = errno; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "Could not stat %s: %s\n", name, errorString); + exit(1); + } + + if (!FastOS_File::MakeDirectory(name)) { + char errorBuf[100]; + int error = errno; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "Could not mkdir(\"%s\", 0775): %s\n", name, errorString); + exit(1); + } +} + +void +FastOS_FileInterface::SetFileName(const char *filename) +{ + if (_filename != NULL) { + free(_filename); + } + + _filename = strdup(filename); +} + + +const char * +FastOS_FileInterface::GetFileName(void) const +{ + return (_filename != NULL) ? _filename : ""; +} + + +bool +FastOS_FileInterface::OpenReadWrite(const char *filename) +{ + return Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_WRITE, filename); +} + + +bool +FastOS_FileInterface::OpenExisting(bool abortIfNotExist, + const char *filename) +{ + bool rc = Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_WRITE | + FASTOS_FILE_OPEN_EXISTING, + filename); + + if (abortIfNotExist && (!rc)) { + std::string errorString = + FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, + "Cannot open %s: %s\n", + filename, + errorString.c_str()); + abort(); + } + + return rc; +} + + +bool +FastOS_FileInterface::OpenReadOnlyExisting(bool abortIfNotExist, + const char *filename) +{ + bool rc = Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_EXISTING, + filename); + + if (abortIfNotExist && (!rc)) { + std::string errorString = + FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, + "Cannot open %s: %s\n", + filename, + errorString.c_str()); + abort(); + } + + return rc; +} + + +bool +FastOS_FileInterface::OpenWriteOnlyTruncate(const char *filename) +{ + // printf("********* OpenWriteOnlyTruncate %s\n", filename); + return Open(FASTOS_FILE_OPEN_WRITE | + FASTOS_FILE_OPEN_CREATE | + FASTOS_FILE_OPEN_TRUNCATE, + filename); +} + + +bool +FastOS_FileInterface::OpenWriteOnlyExisting(bool abortIfNotExist, + const char *filename) +{ + bool rc = Open(FASTOS_FILE_OPEN_WRITE | + FASTOS_FILE_OPEN_EXISTING, + filename); + + if (abortIfNotExist && (!rc)) { + std::string errorString = + FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, + "Cannot open %s: %s\n", + filename, + errorString.c_str()); + abort(); + } + + return rc; +} + +bool +FastOS_FileInterface::OpenReadOnly(const char *filename) +{ + return Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_EXISTING, + filename); +} + + +bool +FastOS_FileInterface::OpenWriteOnly(const char *filename) +{ + return Open(FASTOS_FILE_OPEN_WRITE, filename); +} + + +bool +FastOS_FileInterface::OpenStdin(void) +{ + return Open(FASTOS_FILE_OPEN_STDIN); +} + + +bool +FastOS_FileInterface::OpenStdout(void) +{ + return Open(FASTOS_FILE_OPEN_STDOUT); +} + + +bool +FastOS_FileInterface::OpenStderr(void) +{ + return Open(FASTOS_FILE_OPEN_STDERR); +} + +FastOS_File::Error +FastOS_FileInterface::GetLastError(void) +{ + return FastOS_File::TranslateError(FastOS_File::GetLastOSError()); +} + + +std::string +FastOS_FileInterface::getLastErrorString(void) +{ + int err = FastOS_File::GetLastOSError(); + return FastOS_File::getErrorString(err); +} + +bool FastOS_FileInterface::Rename (const char *newFileName) +{ + bool rc=false; + if (FastOS_File::Rename(GetFileName(), newFileName)) { + SetFileName(newFileName); + rc = true; + } + return rc; +} + +void FastOS_FileInterface::dropFromCache() const +{ +} diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h new file mode 100644 index 00000000000..62481997015 --- /dev/null +++ b/fastos/src/vespa/fastos/file.h @@ -0,0 +1,853 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definitions for FastOS_File, FastOS_DirectoryScan and + * FastOS_StatInfo. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + +#include <vespa/fastos/types.h> +#include <vespa/fastos/mutex.h> +#include <vespa/fastos/time.h> + +#include <vespa/fastos/thread.h> + +#include <string> + +const int FASTOS_FILE_OPEN_READ = (1<<0); +const int FASTOS_FILE_OPEN_WRITE = (1<<1); +const int FASTOS_FILE_OPEN_EXISTING = (1<<2); +const int FASTOS_FILE_OPEN_CREATE = (1<<3); +const int FASTOS_FILE_OPEN_TRUNCATE = (1<<4); +const int FASTOS_FILE_OPEN_STDIN = (1<<5); // Use 2 bits here +const int FASTOS_FILE_OPEN_STDOUT = (2<<5); +const int FASTOS_FILE_OPEN_STDERR = (3<<5); +const int FASTOS_FILE_OPEN_STDFLAGS = (3<<5); +const int FASTOS_FILE_OPEN_DIRECTIO = (1<<7); +const int FASTOS_FILE_OPEN_SHAREREAD = (1<<8); +const int FASTOS_FILE_OPEN_SYNCWRITES = (1<<9); // synchronous writes + +/** + * This class contains regular file-access functionality. + * + * Example (how to read 10 bytes from the file 'hello.txt'): + * + * @code + * void Foo::Bar (void) + * { + * FastOS_File file("hello.txt"); + * + * if(file.OpenReadOnly()) + * { + * char buffer[10]; + * + * if(file.Read(buffer, 10) == 10) + * { + * // Read success + * } + * else + * { + * // Read failure + * } + * } + * else + * { + * // Unable to open file 'hello.txt' + * } + * } + * @endcode + */ + +class DirectIOException : public std::exception +{ +public: + DirectIOException(const char * fileName, const void * buffer, size_t length, int64_t offset); + virtual const char* what() const noexcept { return _what.c_str(); } + const void * getBuffer() const { return _buffer; } + size_t getLength() const { return _length; } + int64_t getOffset() const { return _offset; } + const std::string & getFileName() const { return _fileName; } +private: + std::string _what; + std::string _fileName; + const void * _buffer; + size_t _length; + int64_t _offset; +}; + +class FastOS_FileInterface +{ +private: + FastOS_FileInterface (const FastOS_FileInterface&); + FastOS_FileInterface& operator=(const FastOS_FileInterface&); + + // Default options for madvise used on every file opening. Default is FADV_NORMAL + // Set with setDefaultFAdviseOptions() application wide. + // And setFAdviseOptions() per file. + static int _defaultFAdviseOptions; + int _fAdviseOptions; + size_t _writeChunkSize; + void WriteBufInternal(const void *buffer, size_t length); + +public: + using FailedHandler = void (*)(const char *op, + const char *file, + int error, + int64_t offset, + size_t len, + ssize_t rlen); + +protected: + char *_filename; + unsigned int _openFlags; + bool _directIOEnabled; + bool _syncWritesEnabled; + static FailedHandler _failedHandler; +; + +public: + static int getDefaultFAdviseOptions() { return _defaultFAdviseOptions; } + static void setDefaultFAdviseOptions(int options) { _defaultFAdviseOptions = options; } + int getFAdviseOptions() const { return _fAdviseOptions; } + void setFAdviseOptions(int options) { _fAdviseOptions = options; } + + char *ToString () + { + const char dummy[]="FastOS_File::ToString() deprecated"; + char *str = new char [strlen(dummy) + 1]; + strcpy(str, dummy); + return str; + } + + /** + * Initialize the file class. This is invoked by + * @ref FastOS_Application::Init(). + * @return Boolean success/failure + */ + static bool InitializeClass (void); + + /** + * Cleanup the file class. This is invoked by + * @ref FastOS_Application::Cleanup(). + * @return Boolean success/failure + */ + static bool CleanupClass (void); + + /** + * Set the handler called on fatal file errors. + * @param failedHandler new handler routine. + */ + static void SetFailedHandler(FailedHandler failedHandler) + { + _failedHandler = failedHandler; + } + + /** + * Copy a single file. Will overwrite destination if it already exists. + * + * @author Sveinar Rasmussen + * @return success/failure + * @param src a 'const char *' value with the file to copy from + * @param dst a 'const char *' value with the name of the resulting copy + */ + static bool CopyFile( const char *src, const char *dst ); + + /** + * Move a file from src to dst. Has the same semantics as + * FastOS_File::Rename, except that it works across different + * volumes / disks as well (Via copy and remove). + * + * @author Terje Loken + * @return success / failure + * @param src a 'const char *' value with the file to move from + * @param dst a 'const char *' value with the name of the resulting filename + */ + static bool MoveFile( const char *src, const char *dst); + + /** + * Remove a directory, even if it is non-empty. Missing directory does not cause error. + * + * @author Terje Loken + * @throws std::runtime_error if there are errors. + * @param dir a 'const char *' valuem, with the path to the directory we + * want to remove. + */ + static void EmptyAndRemoveDirectory(const char *dir); + + /** + * Empty a directory. Delete all files and directories within the + * dir, with the exception of files matching a specific name + * (optional). The exception does not apply files in + * subdirectories. + * + * @author Terje Loken + * @throws std::runtime_error if there are errors. + * @param dir a 'const char *' value with the directory to empty. + * @param keepFile a 'const char *' value. If supplied, leave files with + * this name alone. + */ + static void EmptyDirectory( const char *dir, + const char *keepFile = NULL); + + /** + * Make a directory (special compatibility version) + * Succeed if the directory already exists. A stat is performed + * to check the directory before attempting to create the + * directory. + * If the procedure fails, an error is printed to stderr and + * the program exits. + * @param name Name of directory + */ + static void MakeDirIfNotPresentOrExit(const char *name); + + /** + * Return path separator string. This will yield "/" on UNIX systems. + * @return pointer to path separator character string + */ + static const char *GetPathSeparator(void) { return "/";}; + + /** + * Constructor. A filename could be supplied at this point, or specified + * later using @ref SetFileName() or @ref Open(). + * @param filename a filename (optional) + */ + FastOS_FileInterface(const char *filename=NULL); + + /** + * Destructor. If the current file is open, the destructor will close + * it for you. + */ + virtual ~FastOS_FileInterface(void); + + /** + * Associate a new filename with this object. This filename will be + * used when performing @ref Open() (unless @ref Open() specifies a + * different filename). + * @param filename filename character string (will be copied internally) + */ + virtual void SetFileName(const char *filename); + + /** + * Return the filename associated with the File object. If no filename + * has been set, an empty string is returned instead. + * @return The filename associated with the File object + */ + virtual const char *GetFileName(void) const; + + /** + * Open a file. + * @param openFlags A combination of the flags: FASTOS_FILE_OPEN_READ, + * (Is read-access desired?), FASTOS_FILE_OPEN_WRITE + * (Is write-access desired?), and + * FASTOS_FILE_OPEN_EXISTING (The file to be opened + * should already exist. If the file does not exist, + * the call will fail.). + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + virtual bool Open(unsigned int openFlags, const char *filename=NULL) = 0; + + /** + * Open a file for read/write access. The file will be created if it does + * not already exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to @ref Open(). + * @return Boolean success/failure + */ + bool + OpenReadWrite(const char *filename=NULL); + + /** + * Open a file for read/write access. This method fails if the file does + * not already exist. + * @param abortIfNotExist Abort the program if the file does not exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool + OpenExisting(bool abortIfNotExist=false, const char *filename=NULL); + + /** + * Open a file for read access. This method fails if the file does + * not already exist. + * @param abortIfNotExist Abort the program if the file does not exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool + OpenReadOnlyExisting (bool abortIfNotExist=false, + const char *filename=NULL); + + /** + * Open a file for write access. If the file does not exist, it is created. + * If the file exists, it is truncated to 0 bytes. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool + OpenWriteOnlyTruncate(const char *filename=NULL); + + /** + * Open a file for write access. This method fails if the file does + * not already exist. + * @param abortIfNotExist Abort the program if the file does not exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool + OpenWriteOnlyExisting (bool abortIfNotExist=false, + const char *filename=NULL); + + /** + * Open a file for read-access only. This method fails if the file does + * not already exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to @ref Open(). + * @return Boolean success/failure + */ + bool + OpenReadOnly(const char *filename=NULL); + + /** + * Open a file for write-access only. The file will be created if it does + * not already exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to @ref Open(). + * @return Boolean success/failure + */ + bool + OpenWriteOnly(const char *filename=NULL); + + bool + OpenStdin(void); + + bool + OpenStdout(void); + + bool + OpenStderr(void); + + /** + * Close the file. The call will successfully do nothing if the file + * already is closed. + * @return Boolean success/failure + */ + virtual bool Close(void) = 0; + + /** + * Is the file currently opened? + * @return true if the file is opened, else false + */ + virtual bool IsOpened(void) const = 0; + + /** + * Read [length] bytes into [buffer]. + * @param buffer buffer pointer + * @param length number of bytes to read + * @return The number of bytes which was actually read, + * or -1 on error. + */ + virtual ssize_t Read(void *buffer, size_t length) = 0; + + /** + * Write [len] bytes from [buffer]. This is just a wrapper for + * Write2, which does the same thing but returns the number of + * bytes written instead of just a bool. + * @param buffer buffer pointer + * @param len number of bytes to write + * @return Boolean success/failure + */ + bool CheckedWrite(const void *buffer, size_t len); + + /** + * Write [len] bytes from [buffer]. + * @param buffer buffer pointer + * @param len number of bytes to write + * @return The number of bytes actually written, or -1 on error + */ + virtual ssize_t Write2(const void *buffer, size_t len) = 0; + + /** + * Read [length] bytes into [buffer]. Caution! If the actual number + * of bytes read != [length], an error message is printed to stderr + * and the program aborts. + * + * The method is included for backwards compatibility reasons. + * @param buffer buffer pointer + * @param length number of bytes to read + */ + virtual void ReadBuf(void *buffer, size_t length); + + /** + * Write [length] bytes from [buffer] in chunks. Caution! If the write fails, + * an error message is printed to stderr and the program aborts. + * + * The method is included for backwards compatibility reasons. + * @param buffer buffer pointer + * @param length number of bytes to write + */ + virtual void WriteBuf(const void *buffer, size_t length); + + + /** + * Read [length] bytes at file offset [readOffset] into [buffer]. + * Only thread-safe if an OS-specific implementation exists. + * + * Caution! If the actual number of bytes read != [length], an + * error message is printed to stderr and the program aborts. + * + * @param buffer buffer pointer + * @param length number of bytes to read + * @param readOffset file offset where the read operation starts + */ + virtual void ReadBuf(void *buffer, size_t length, int64_t readOffset); + + /** + * Set the filepointer. The next @ref Read() or @ref Write() will + * continue from this position. + * @param position position of the new file pointer (in bytes) + * @return Boolean success/failure + */ + virtual bool SetPosition(int64_t position) = 0; + + /** + * Get the filepointer. -1 is returned if the operation fails. + * @return current position of file pointer (in bytes) + */ + virtual int64_t GetPosition(void) = 0; + + /** + * const version of @link GetPosition + */ + int64_t getPosition(void) const { return const_cast<FastOS_FileInterface *>(this)->GetPosition(); } + + /** + * Return the file size. This method requires that the file is + * currently opened. If you wish to determine the file size + * without opening the file, use @ref Stat(). + * If an error occurs, the returned value is -1. + * @return file size (in bytes) + */ + virtual int64_t GetSize(void) = 0; + + /** + * const version of @link GetSize + */ + int64_t getSize(void) const { return const_cast<FastOS_FileInterface *>(this)->GetSize(); } + + /** + * Return the time when file was last modified. + * @return time of last modification + */ + virtual time_t GetModificationTime(void) = 0; + + /** + * Delete the file. This method requires that the file is + * currently not opened. + * @return Boolean success/failure + */ + virtual bool Delete(void) = 0; + + /** + * Rename/move a file or directory. This method requires that + * the file is currently not opened. A move operation is + * supported as long as the source and destination reside + * on the same volume/device. + * The method fails if the destination already exists. + * @param newFileName New file name + * @return Boolean success/failure + */ + virtual bool Rename (const char *newFileName); + + /** + * Force completion of pending disk writes (flush cache). + */ + virtual bool Sync(void) = 0; + + /** + * Are we in some kind of file read mode? + */ + bool IsReadMode(void) + { + return ((_openFlags & FASTOS_FILE_OPEN_READ) != 0); + } + + /** + * Are we in some kind of file write mode? + */ + bool IsWriteMode(void) + { + return ((_openFlags & FASTOS_FILE_OPEN_WRITE) != 0); + } + + /** + * Truncate or extend the file to the new size. + * The file pointer is also set to [newSize]. + * @ref SetSize() requires write access to the file. + * @param newSize New file size + * @return Boolean success/failure. + */ + virtual bool SetSize(int64_t newSize) = 0; + + /** + * Enable direct disk I/O (disable OS buffering & cache). Reads + * and writes will be performed directly to or from the user + * program buffer, provided appropriate size and alignment + * restrictions are met. If the restrictions are not met, a + * normal read or write is performed as a fallback. + * Call this before opening a file, and query + * @ref GetDirectIORestrictions() after a file is opened to get the + * neccessary alignment restrictions. It is possible that direct + * disk I/O could not be enabled. In that case + * @ref GetDirectIORestrictions will return false. + */ + virtual void EnableDirectIO(void); + + virtual void + EnableSyncWrites(void); + + /** + * Set the write chunk size used in WriteBuf. + */ + void + SetWriteChunkSize(size_t writeChunkSize); + + /** + * Get restrictions for direct disk I/O. The file should be opened + * before this method is called. + * Even though direct disk I/O is enabled through @ref EnableDirectIO(), + * this method could still return false, indicating that direct disk I/O + * is not being used for this file. This could be caused by either: no + * OS support for direct disk I/O, direct disk I/O might only be implemented + * for certain access-modes, the file is on a network drive or other + * partition where direct disk I/O is disallowed. + * + * The restriction-arguments are always filled with valid data, independant + * of the return code and direct disk I/O availability. + * + * @param memoryAlignment Buffer alignment restriction + * @param transferGranularity All transfers must be a multiple of + * [transferGranularity] bytes. All + * file offsets for these transfers must + * also be a multiple of [transferGranularity] + * bytes. + * @param transferMaximum All transfers must be <= [transferMaximum] + * bytes. + * @return True if direct disk I/O is being used for this file, else false. + */ + virtual bool + GetDirectIORestrictions(size_t &memoryAlignment, + size_t &transferGranularity, + size_t &transferMaximum); + + /** + * Retrieve the required padding for direct I/O to be used. + * + * @param offset File offset + * @param buflen Buffer length + * @param padBefore Number of pad bytes needed in front of the buffer + * @param padAfter Number of pad bytes needed after the buffer + * + * @return True if the file access can be accomplished with + * direct I/O, else false. + */ + virtual bool DirectIOPadding(int64_t offset, + size_t buflen, + size_t &padBefore, + size_t &padAfter); + + /** + * Allocate a buffer properly alligned with regards to direct io + * access restrictions. + * @param byteSize Number of bytes to be allocated + * @param realPtr Reference where the actual pointer returned + * from malloc will be saved. Use free() with + * this pointer to deallocate the buffer. + * This value is always set. + * @return Alligned pointer value or NULL if out of memory + */ + virtual void *AllocateDirectIOBuffer(size_t byteSize, void *&realPtr); + + /** + * Enable mapping of complete file contents into the address space of the + * running process. This only works for small files. This operation + * will be ignored on some file types. + */ + virtual void enableMemoryMap(int mmapFlags); + + /** + * Inquiry about where in memory file data is located. + * @return location of file data in memory. If the file is not mapped, + * NULL is returned. + */ + virtual void *MemoryMapPtr(int64_t position) const; + + /** + * Inquiry if file content is mapped into memory. + * @return true if file is mapped in memory, false otherwise. + */ + virtual bool IsMemoryMapped(void) const; + + /** + * Will drop whatever is in the FS cache when called. Does not have effect in the future. + **/ + virtual void dropFromCache() const; + + enum Error + { + ERR_ZERO = 1, // No error New style + ERR_NOENT, // No such file or directory + ERR_NOMEM, // Not enough memory + ERR_ACCES, // Permission denied + ERR_EXIST, // File exists + ERR_INVAL, // Invalid argument + ERR_NFILE, // File table overflow + ERR_MFILE, // Too many open files + ERR_NOSPC, // No space left on device + ERR_INTR, // interrupt + ERR_AGAIN, // Resource unavailable, try again + ERR_BUSY, // Device or resource busy + ERR_IO, // I/O error + ERR_PERM, // Not owner + ERR_NODEV, // No such device + ERR_NXIO, // Device not configured + ERR_UNKNOWN, // Unknown + + ERR_EZERO = 1, // No error Old style + ERR_ENOENT, // No such file or directory + ERR_ENOMEM, // Not enough memory + ERR_EACCES, // Permission denied + ERR_EEXIST, // File exists + ERR_EINVAL, // Invalid argument + ERR_ENFILE, // File table overflow + ERR_EMFILE, // Too many open files + ERR_ENOSPC, // No space left on device + ERR_EINTR, // interrupt + ERR_EAGAIN, // Resource unavailable, try again + ERR_EBUSY, // Device or resource busy + ERR_EIO, // I/O error + ERR_EPERM, // Not owner + ERR_ENODEV, // No such device + ERR_ENXIO // Device not configured + }; + + + /** + * If a file operation fails, the error code can be retrieved + * via this method. See @ref Error for possible error codes. + * @return Error code + */ + static Error GetLastError(void); + + /** + * Similar to @ref GetLastError(), but this method returns a string + * instead of an error code. + * @return String describing the last error + */ + static std::string + getLastErrorString(void); +}; + + +/** + * The class serves as a container for information returned by + * @ref FastOS_File::Stat(). + */ +class FastOS_StatInfo +{ +public: + /** + * Possible error codes. + */ + enum StatError + { + Ok, //!< ok + Unknown, //!< unknown error + FileNotFound //!< file not found error + }; + + StatError _error; + + /** + * Is it a regular file? This field is only valid if @ref _error is + * @ref Ok. + */ + bool _isRegular; + + /** + * Is it a directory? This field is only valid if @ref _error is + * @ref Ok. + */ + bool _isDirectory; + + /** + * File size. This field is only valid if @ref _error is + * @ref Ok. + */ + int64_t _size; + + /** + * Time of last modification in seconds. + */ + time_t _modifiedTime; + + /** + * Time of last modification in seconds. + */ + uint64_t _modifiedTimeNS; +}; + + +/** + * This class enumerates the contents of a given directory. + * + * Example: + * @code + * void Foo::Bar(void) + * { + * // Scan and print the contents of the directory '/usr/src/include' + * + * FastOS_DirectoryScan dirScan("/usr/src/include"); + * int numEntries = 0; + * + * while(dirScan.ReadNext()) + * { + * const char *name = dirScan.GetName(); + * bool isDirectory = dirScan.IsDirectory(); + * bool isRegular = dirScan.IsRegular(); + * + * printf("%-30s %s\n", name, + * isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); + * + * numEntries++; + * } + * + * printf("The directory contained %d entries.\n", numEntries); + * } + * @endcode + */ +class FastOS_DirectoryScanInterface +{ +private: + FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&); + FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&); + +protected: + char *_searchPath; + +public: + + /** + * Constructor. + * + * @param path Path of the directory to be scanned. The path string + * is copied internally. + */ + FastOS_DirectoryScanInterface(const char *path) + : _searchPath(strdup(path)) + { + } + + /** + * Destructor. + * + * Frees operating system resources related to the directory scan. + */ + virtual ~FastOS_DirectoryScanInterface(void) + { + free(_searchPath); + } + + /** + * Get search path. + * This is an internal copy of the path specified in the constructor. + * @return Search path string. + */ + const char *GetSearchPath () { return _searchPath; } + + /** + * Read the next entry in the directory scan. Failure indicates + * that there are no more entries. If the call is successful, + * attributes for the entry can be read with @ref IsDirectory(), + * @ref IsRegular() and @ref GetName(). + * + * @return Boolean success/failure + */ + virtual bool ReadNext(void) = 0; + + /** + * After a successful @ref ReadNext() this method is used to + * determine if the entry is a directory entry or not. Calling this + * method after an unsuccessful @ref ReadNext() or before + * @ref ReadNext() is called for the first time, yields undefined + * results. + * + * @return True if the entry is a directory, else false. + */ + virtual bool IsDirectory(void) = 0; + + + /** + * After a successful @ref ReadNext() this method is used to + * determine if the entry is a regular file entry or not. Calling + * this method after an unsuccessful @ref ReadNext() or before + * @ref ReadNext() is called for the first time, yields undefined + * results. + * + * @return True if the entry is a regular file, else false. + */ + virtual bool IsRegular(void) = 0; + + /** + * After a successful @ref ReadNext() this method is used to + * determine the name of the recently read directory entry. Calling + * this method after an unsuccessful @ref ReadNext() or before + * @ref ReadNext() is called for the first time, yields undefined + * results. + * + * @return A pointer to the recently read directory entry. + */ + virtual const char *GetName(void) = 0; + + /** + * Check whether the creation of a directory scan succeeded or + * failed (e.g. due to resource constraints). + * + * return True if the directory scan is valid. + */ + virtual bool IsValidScan(void) const = 0; +}; + +#include <vespa/fastos/linux_file.h> +typedef FastOS_Linux_File FASTOS_PREFIX(File); +typedef FastOS_UNIX_DirectoryScan FASTOS_PREFIX(DirectoryScan); diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/fastos/src/vespa/fastos/linux_file.cpp new file mode 100644 index 00000000000..c78551180f6 --- /dev/null +++ b/fastos/src/vespa/fastos/linux_file.cpp @@ -0,0 +1,444 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-09-21 +* @file +* Implementation of FastOS_Linux_File methods. +*****************************************************************************/ + +#include <vespa/fastos/file.h> +#include <sstream> +#include <stdexcept> + +const size_t FastOS_Linux_File::_directIOFileAlign = 4096; +const size_t FastOS_Linux_File::_directIOMemAlign = 4096; +const size_t FastOS_Linux_File::_pageSize = 4096; + +FastOS_Linux_File::FastOS_Linux_File(const char *filename) + : FastOS_UNIX_File(filename), + _cachedSize(-1), + _filePointer(-1) +{ +} + +#define DIRECTIOPOSSIBLE(buf, len, off) \ + ((off & (_directIOFileAlign - 1)) == 0 && \ + (len & (_directIOFileAlign - 1)) == 0 && \ + (reinterpret_cast<unsigned long>(buf) & (_directIOMemAlign - 1)) == 0) + +ssize_t +FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length, + int64_t readOffset) +{ + ssize_t readResult = ::pread(fh, buffer, length, readOffset); + if (readResult < 0 && _failedHandler != NULL) { + int error = errno; + const char *fileName = GetFileName(); + _failedHandler("read", + fileName, + error, + readOffset, + length, + readResult); + errno = error; + } + return readResult; +} + + +ssize_t +FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length) +{ + ssize_t readResult = ::read(fh, buffer, length); + if (readResult < 0 && _failedHandler != NULL) { + int error = errno; + int64_t readOffset = GetPosition(); + const char *fileName = GetFileName(); + _failedHandler("read", + fileName, + error, + readOffset, + length, + readResult); + errno = error; + } + return readResult; +} + + +ssize_t +FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length, + int64_t writeOffset) +{ + ssize_t writeRes = ::pwrite(fh, buffer, length, writeOffset); + if (writeRes < 0 && _failedHandler != NULL) { + int error = errno; + const char *fileName = GetFileName(); + _failedHandler("write", + fileName, + error, + writeOffset, + length, + writeRes); + errno = error; + } + return writeRes; +} + +ssize_t +FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length) +{ + ssize_t writeRes = ::write(fh, buffer, length); + if (writeRes < 0 && _failedHandler != NULL) { + int error = errno; + int64_t writeOffset = GetPosition(); + const char *fileName = GetFileName(); + _failedHandler("write", + fileName, + error, + writeOffset, + length, + writeRes); + errno = error; + } + return writeRes; +} + + +ssize_t FastOS_Linux_File::readUnalignedEnd(void *buffer, size_t length, int64_t readOffset) +{ + if (length == 0) { return 0; } + int fh = open(GetFileName(), O_RDONLY, 0664); + if (fh < 0) { + std::ostringstream os; + os << "Failed opening file " << GetFileName() << " for reading the unaligend end due to : " << getLastErrorString(); + throw std::runtime_error(os.str()); + } + ssize_t readResult = readInternal(fh, buffer, length, readOffset); + close(fh); + return readResult; +} + +ssize_t FastOS_Linux_File::writeUnalignedEnd(const void *buffer, size_t length, int64_t writeOffset) +{ + if (length == 0) { return 0; } + int fh = open(GetFileName(), O_WRONLY | O_SYNC, 0664); + if (fh < 0) { + std::ostringstream os; + os << "Failed opening file " << GetFileName() << " for reading the unaligend end due to : " << getLastErrorString(); + throw std::runtime_error(os.str()); + } + ssize_t writeResult = writeInternal(fh, buffer, length, writeOffset); + close(fh); + return writeResult; +} + +ssize_t +FastOS_Linux_File::ReadBufInternal(void *buffer, size_t length, int64_t readOffset) +{ + if (length == 0) { return 0; } + ssize_t readResult; + + if (_directIOEnabled) { + if (DIRECTIOPOSSIBLE(buffer, length, readOffset)) { + readResult = readInternal(_filedes, buffer, length, readOffset); + } else { + size_t alignedLength(length & ~(_directIOFileAlign - 1)); + if (DIRECTIOPOSSIBLE(buffer, alignedLength, readOffset)) { + size_t remain(length - alignedLength); + if (alignedLength > 0) { + readResult = readInternal(_filedes, buffer, alignedLength, + readOffset); + } else { + readResult = 0; + } + if (static_cast<size_t>(readResult) == alignedLength && + remain != 0) { + ssize_t readResult2 = readUnalignedEnd( + static_cast<char *>(buffer) + alignedLength, + remain, readOffset + alignedLength); + if (readResult == 0) { + readResult = readResult2; + } else if (readResult2 > 0) { + readResult += readResult2; + } + } + } else { + throw DirectIOException(GetFileName(), buffer, length, readOffset); + } + } + } else { + readResult = readInternal(_filedes, buffer, length, readOffset); + } + + if (readResult < 0) { + perror("pread error"); + } + + return readResult; +} + +void +FastOS_Linux_File::ReadBuf(void *buffer, size_t length, int64_t readOffset) +{ + ssize_t readResult(ReadBufInternal(buffer, length, readOffset)); + if (static_cast<size_t>(readResult) != length) { + std::string errorString = (readResult != -1) + ? std::string("short read") + : getLastErrorString(); + std::ostringstream os; + os << "Fatal: Reading " << length << " bytes, got " << readResult << " from '" + << GetFileName() << "' failed: " << errorString; + throw std::runtime_error(os.str()); + } +} + + +ssize_t +FastOS_Linux_File::Read(void *buffer, size_t len) +{ + if (_directIOEnabled) { + ssize_t readResult = ReadBufInternal(buffer, len, _filePointer); + if (readResult > 0) { + _filePointer += readResult; + } + return readResult; + } else { + return readInternal(_filedes, buffer, len); + } +} + + +ssize_t +FastOS_Linux_File::Write2(const void *buffer, size_t length) +{ + if (length == 0) { return 0; } + ssize_t writeRes; + + if (_directIOEnabled) { + if (DIRECTIOPOSSIBLE(buffer, length, _filePointer)) { + writeRes = writeInternal(_filedes, buffer, length, _filePointer); + } else { + size_t alignedLength(length & ~(_directIOFileAlign - 1)); + if (DIRECTIOPOSSIBLE(buffer, alignedLength, _filePointer)) { + size_t remain(length - alignedLength); + if (alignedLength > 0) { + writeRes = writeInternal(_filedes, buffer, alignedLength, + _filePointer); + } else { + writeRes = 0; + } + if (static_cast<size_t>(writeRes) == alignedLength && + remain != 0) { + ssize_t writeRes2 = writeUnalignedEnd( + static_cast<const char *>(buffer) + alignedLength, + remain, _filePointer + alignedLength); + if (writeRes == 0) { + writeRes = writeRes2; + } else if (writeRes2 > 0) { + writeRes += writeRes2; + } + } + } else { + throw DirectIOException(GetFileName(), buffer, length, _filePointer); + } + } + if (writeRes > 0) { + _filePointer += writeRes; + if (_filePointer > _cachedSize) { + _cachedSize = _filePointer; + } + } + } else { + writeRes = writeInternal(_filedes, buffer, length); + } + + return writeRes; +} + + +bool +FastOS_Linux_File::SetPosition(int64_t desiredPosition) +{ + bool rc = FastOS_UNIX_File::SetPosition(desiredPosition); + + if (rc && _directIOEnabled) { + _filePointer = desiredPosition; + } + + return rc; +} + + +int64_t +FastOS_Linux_File::GetPosition(void) +{ + return _directIOEnabled ? _filePointer : FastOS_UNIX_File::GetPosition(); +} + + +bool +FastOS_Linux_File::SetSize(int64_t newSize) +{ + bool rc = FastOS_UNIX_File::SetSize(newSize); + + if (rc) { + _cachedSize = newSize; + } + return rc; +} + + +namespace { + void * align(void * p, size_t alignment) { + const size_t alignMask(alignment-1); + return reinterpret_cast<void *>((reinterpret_cast<unsigned long>(p) + alignMask) & ~alignMask); + } +} + +void * +FastOS_Linux_File::AllocateDirectIOBuffer (size_t byteSize, void *&realPtr) +{ + size_t dummy1, dummy2; + size_t memoryAlignment; + + GetDirectIORestrictions(memoryAlignment, dummy1, dummy2); + + realPtr = malloc(byteSize + memoryAlignment - 1); + return align(realPtr, memoryAlignment); +} + + +void * +FastOS_Linux_File:: +allocateGenericDirectIOBuffer(size_t byteSize, + void *&realPtr) +{ + size_t memoryAlignment = _directIOMemAlign; + realPtr = malloc(byteSize + memoryAlignment - 1); + return align(realPtr, memoryAlignment); +} + + +size_t +FastOS_Linux_File::getMaxDirectIOMemAlign(void) +{ + return _directIOMemAlign; +} + + +bool +FastOS_Linux_File::GetDirectIORestrictions (size_t &memoryAlignment, + size_t &transferGranularity, + size_t &transferMaximum) +{ + bool rc = false; + + if (_directIOEnabled) { + memoryAlignment = _directIOMemAlign; + transferGranularity = _directIOFileAlign; + transferMaximum = 0x7FFFFFFF; + rc = true; + } else { + rc = FastOS_UNIX_File::GetDirectIORestrictions(memoryAlignment, transferGranularity, transferMaximum); + } + + return rc; +} + + +bool +FastOS_Linux_File::DirectIOPadding (int64_t offset, + size_t length, + size_t &padBefore, + size_t &padAfter) +{ + if (_directIOEnabled) { + padBefore = offset & (_directIOFileAlign - 1); + padAfter = _directIOFileAlign - ((padBefore + length) & (_directIOFileAlign - 1)); + + if (padAfter == _directIOFileAlign) { + padAfter = 0; + } else if ((static_cast<int64_t>(offset + length + padAfter) > _cachedSize) && + (static_cast<int64_t>(offset + length) <= _cachedSize)) { + padAfter = _cachedSize - offset - length; + } + + if (static_cast<uint64_t>(offset + length + padAfter) <= static_cast<uint64_t>(_cachedSize)) { + return true; + } + } + + padAfter = 0; + padBefore = 0; + + return false; +} + + +void +FastOS_Linux_File::EnableDirectIO(void) +{ + if (!IsOpened()) { + _directIOEnabled = true; + } +} + + +bool +FastOS_Linux_File::Open(unsigned int openFlags, const char *filename) +{ + bool rc; + _cachedSize = -1; + _filePointer = -1; + if (_directIOEnabled && (_openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { + _directIOEnabled = false; + } + if (_syncWritesEnabled) + openFlags |= FASTOS_FILE_OPEN_SYNCWRITES; + if (_directIOEnabled) { + rc = FastOS_UNIX_File::Open(openFlags | FASTOS_FILE_OPEN_DIRECTIO, filename); + if ( ! rc ) { //Retry without directIO. + rc = FastOS_UNIX_File::Open(openFlags | FASTOS_FILE_OPEN_SYNCWRITES, filename); + } + if (rc) { + int fadviseOptions = getFAdviseOptions(); + if (POSIX_FADV_NORMAL != fadviseOptions) { + rc = (posix_fadvise(_filedes, 0, 0, fadviseOptions) == 0); + if (!rc) { + Close(); + } + } + } + if (rc) { + Sync(); + _cachedSize = GetSize(); + _filePointer = 0; + } + } else { + rc = FastOS_UNIX_File::Open(openFlags, filename); + if (rc && (POSIX_FADV_NORMAL != getFAdviseOptions())) { + rc = (posix_fadvise(_filedes, 0, 0, getFAdviseOptions()) == 0); + if (!rc) { + Close(); + } + } + } + return rc; +} + + +bool +FastOS_Linux_File::InitializeClass(void) +{ + return FastOS_UNIX_File::InitializeClass(); +} + +#include <vespa/fastos/backtrace.h> + +void forceStaticLinkOf_backtrace() +{ + void * dummy[2]; + FastOS_backtrace(dummy, 2); +} diff --git a/fastos/src/vespa/fastos/linux_file.h b/fastos/src/vespa/fastos/linux_file.h new file mode 100644 index 00000000000..5dcf4eaf7e3 --- /dev/null +++ b/fastos/src/vespa/fastos/linux_file.h @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-09-21 +* @file +* Class definition for FastOS_Linux_File. +*****************************************************************************/ + +#pragma once + +#include <vespa/fastos/unix_file.h> + +/** + * This is the Linux implementation of @ref FastOS_File. Most + * methods are inherited from @ref FastOS_UNIX_File. + */ +class FastOS_Linux_File : public FastOS_UNIX_File +{ +public: + using FastOS_UNIX_File::ReadBuf; +protected: + int64_t _cachedSize; + int64_t _filePointer; // Only maintained/used in directio mode + +public: + FastOS_Linux_File (const char *filename = NULL); + virtual ~FastOS_Linux_File () { + Close(); + } + + virtual bool + GetDirectIORestrictions(size_t &memoryAlignment, + size_t &transferGranularity, + size_t &transferMaximum); + + virtual bool + DirectIOPadding(int64_t offset, size_t length, + size_t &padBefore, + size_t &padAfter); + + virtual void + EnableDirectIO(void); + + virtual bool + SetPosition(int64_t desiredPosition); + + virtual int64_t + GetPosition(void); + + virtual bool + SetSize(int64_t newSize); + + virtual void + ReadBuf(void *buffer, size_t length, int64_t readOffset); + + virtual void * + AllocateDirectIOBuffer(size_t byteSize, void *&realPtr); + + static void * + allocateGenericDirectIOBuffer(size_t byteSize, + void *&realPtr); + + static size_t + getMaxDirectIOMemAlign(void); + + virtual ssize_t + Read(void *buffer, size_t len); + + virtual ssize_t + Write2(const void *buffer, size_t len); + + virtual bool + Open(unsigned int openFlags, const char *filename = NULL); + + static bool + InitializeClass(void); +private: + ssize_t + readUnalignedEnd(void *buffer, size_t length, int64_t readOffset); + ssize_t + writeUnalignedEnd(const void *buffer, size_t length, int64_t readOffset); + ssize_t + ReadBufInternal(void *buffer, size_t length, int64_t readOffset); + + ssize_t + readInternal(int fh, void *buffer, size_t length, int64_t readOffset); + + ssize_t + readInternal(int fh, void *buffer, size_t length); + + ssize_t + writeInternal(int fh, const void *buffer, size_t length, + int64_t writeOffset); + + ssize_t + writeInternal(int fh, const void *buffer, size_t length); + + static const size_t _directIOFileAlign; + static const size_t _directIOMemAlign; + static const size_t _pageSize; +}; diff --git a/fastos/src/vespa/fastos/mutex.h b/fastos/src/vespa/fastos/mutex.h new file mode 100644 index 00000000000..daea908eed5 --- /dev/null +++ b/fastos/src/vespa/fastos/mutex.h @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definition for FastOS_Mutex. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + +#include <vespa/fastos/types.h> + + +/** + * This class defines a mutual-exclusion object. + * + * Facilitates synchronized access to mutual-exclusion zones in the program. + * Before entering code sections where only a single thread at the time can + * operate, use @ref Lock(). If another thread is holding the lock at the + * time, the calling thread will sleep until the current holder of the mutex + * is through using it. + * + * Use @ref Unlock() to release the mutex lock. This will allow other threads + * to obtain the lock. + */ + +class FastOS_MutexInterface +{ +public: + /** + * Destructor + */ + virtual ~FastOS_MutexInterface () {}; + + /** + * Obtain an exclusive lock on the mutex. The result of a recursive lock + * is currently undefined. The caller should assume this will result + * in a deadlock situation. + * A recursive lock occurs when a thread, currently owning the lock, + * attempts to lock the mutex a second time. + * + * Use @ref Unlock() to unlock the mutex when done. + */ + virtual void Lock (void)=0; + + /** + * Try to obtain an exclusive lock on the mutex. If a lock cannot be + * obtained right away, the method will return false. There will + * be no blocking/waiting for the mutex lock to be available. If + * the mutex was locked in the attempt, true is returned. + * @return Boolean success/failure + */ + virtual bool TryLock (void)=0; + + /** + * Unlock a locked mutex. The result of unlocking a mutex not already + * locked by the calling thread is undefined. + */ + virtual void Unlock (void)=0; +}; + +#include <vespa/fastos/unix_mutex.h> +typedef FastOS_UNIX_Mutex FASTOS_PREFIX(Mutex); + diff --git a/fastos/src/vespa/fastos/prefetch.h b/fastos/src/vespa/fastos/prefetch.h new file mode 100644 index 00000000000..a9b854454be --- /dev/null +++ b/fastos/src/vespa/fastos/prefetch.h @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definition for FastOS_PrefetchInterface. + * + * @author Olaf Birkeland + */ + +#pragma once + +// If a known architecture/compiler is found, PREFETCH_IMPLEMENTED is +// defined. + + +/** + * The prefetch functions are used to improve cache management. They + * will all try to bring data into cache before actually being needed by the + * code, thus avoiding cache misses. All functions take the address of the + * element to be prefetched as a parameter. An illegal address will NOT cause + * any exceptions, thus prefetch can be used speculatively. + * <p> + * Prefetch is always done in entire cache line sizes. The cache line size is + * platform dependent, and defined by the method @ref GetPrefetchSize() + * (in bytes), usually 32 or 64 bytes. Use this constant to scale the + * unroll-factor of loops etc. Prefetch is applied to the cache line + * surrounding the argument address, thus the argument does not need + * to be aligned in any way. + * <p> + * Prefetch has no side effects, and can be omitted without altering functional + * code behavior (e.g. for easier debugging). Not all implementations have all + * the defined cache levels, thus some functions will be aliases on some + * platforms. + * <p> + * Performance considerations:<br> + * <ul> + * <li>Prefetch only once for each cache line. Manual loop unrolling is thus + * likely to be needed.</li> + * <li>How far ahead prefetch should be used is platform and algorithm + * dependent.</li> + * <li>Prefetching too early can potentially decrease performance due to cache + * trashing.</li></ul> + */ +class FastOS_PrefetchInterface +{ +public: + virtual ~FastOS_PrefetchInterface(void) { } +}; + +#ifdef GCC_X86_64 +# include "prefetch_gcc_x86_64.h" +typedef FastOS_GCCX86_64_Prefetch FASTOS_PREFIX(Prefetch); +#else +/** + * Default fallback for unsupported architecture/compilers + */ +class FastOS_Dummy_Prefetch : public FastOS_PrefetchInterface +{ +public: + inline static int GetPrefetchSize () { return 32; }; + inline static void L0(const void *data) { (void)data; }; + inline static void L1(const void *data) { (void)data; }; + inline static void L2(const void *data) { (void)data; }; + inline static void NT(const void *data) { (void)data; }; +}; +typedef FastOS_Dummy_Prefetch FASTOS_PREFIX(Prefetch); + +#endif + diff --git a/fastos/src/vespa/fastos/prefetch_gcc_sparc.h b/fastos/src/vespa/fastos/prefetch_gcc_sparc.h new file mode 100644 index 00000000000..019f3c00d47 --- /dev/null +++ b/fastos/src/vespa/fastos/prefetch_gcc_sparc.h @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Class definition for FastOS_GCCSPARC_Prefetch. + * + * @author Olaf Birkeland + */ + +#pragma once + + +// SPARC on GCC +class FastOS_GCCSPARC_Prefetch : public FastOS_PrefetchInterface +{ + enum Constants + { + PREFETCH_SIZE = 64 + }; + +public: + inline static int GetPrefetchSize () { return PREFETCH_SIZE; } + + inline static void L0 (const void *data) + { + __asm__ ("prefetch [%0], 0" : : "r" (data)); + } + inline static void L1 (const void *data) + { + __asm__ ("prefetch [%0], 0" : : "r" (data)); + } + inline static void L2 (const void *data) + { + __asm__ ("prefetch [%0], 4" : : "r" (data)); + } + inline static void NT (const void *data) + { + __asm__ ("prefetch [%0], 1" : : "r" (data)); + } +}; + + diff --git a/fastos/src/vespa/fastos/prefetch_gcc_x86_64.h b/fastos/src/vespa/fastos/prefetch_gcc_x86_64.h new file mode 100644 index 00000000000..76292fc83a3 --- /dev/null +++ b/fastos/src/vespa/fastos/prefetch_gcc_x86_64.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Class definition for FastOS_GCCX86_64__Prefetch. + * + * @author Olaf Birkeland + */ + +#pragma once + + +// x86 on GCC +class FastOS_GCCX86_64_Prefetch : public FastOS_PrefetchInterface +{ + enum Constants + { + PREFETCH_SIZE = 64 + }; + +public: + inline static int GetPrefetchSize () { return PREFETCH_SIZE; } + + inline static void L0 (const void *data) + { + __asm__("prefetcht0 %0" : : "m" (data)); + } + + inline static void L1 (const void *data) + { + __asm__("prefetcht1 %0" : : "m" (data)); + } + + inline static void L2 (const void *data) + { + __asm__("prefetcht2 %0" : : "m" (data)); + } + + inline static void NT (const void *data) + { + __asm__("prefetchnta %0" : : "m" (data)); + } +}; + + diff --git a/fastos/src/vespa/fastos/process.h b/fastos/src/vespa/fastos/process.h new file mode 100644 index 00000000000..01e88761704 --- /dev/null +++ b/fastos/src/vespa/fastos/process.h @@ -0,0 +1,310 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definitions for FastOS_ProcessInterface and + * FastOS_ProcessRedirectListener. + * + * @author Oivind H. Danielsen + */ + +#pragma once + +#include <vespa/fastos/types.h> + +/** + * This class serves as a sink for redirected (piped) output from + * subclasses of @ref FastOS_ProcessInterface. + */ +class FastOS_ProcessRedirectListener +{ +public: + /** + * This method is called when new data is available from the + * process. Subclass this method to process the data. + * You should assume that any thread can invoke this method. + * For convenience the data buffer is always zero- terminated + * (static_cast<uint8_t>(data[length]) = '\\0'). + * When the pipe closes, the method is invoked with data = NULL + * and length = 0. + * @param data Pointer to data + * @param length Length of data block in bytes + */ + virtual void OnReceiveData (const void *data, size_t length) = 0; + + virtual ~FastOS_ProcessRedirectListener () {} +}; + +class FastOS_ThreadPool; +class FastOS_ApplicationInterface; + +/** + * This class can start a process, redirect standard input, output + * and error streams, kill process, wait for process to exit + * and send IPC messages to the process. + */ +class FastOS_ProcessInterface +{ +private: + FastOS_ProcessInterface(const FastOS_ProcessInterface&); + FastOS_ProcessInterface &operator=(const FastOS_ProcessInterface &); + +protected: + // Hack to achieve 64-bit alignment of + // FastOS_ProcessInterface pointers (for Sparc) + double _extradoublehackforalignment; + + char *_cmdLine; + bool _pipeStdin; + + FastOS_ProcessRedirectListener *_stdoutListener; + FastOS_ProcessRedirectListener *_stderrListener; + + int _bufferSize; + + FastOS_ThreadPool *GetThreadPool (); + +public: + FastOS_ProcessInterface *_next, *_prev; + static FastOS_ApplicationInterface *_app; + + /** + * Call this prior to opening files with fopen to avoid having + * these file handles inherited by child processes. Remember + * to call PostfopenNoInherit(x) after fopen. x is the return value + * of PrefopenNoInherit. + * @return Internal data needed to cancel object inheritance. + */ + static void *PrefopenNoInherit (void) {return NULL;} + + /** + * Call this after opening files with fopen to avoid having + * these files handles inherited by child processes. Remember + * to call PrefopenNoInherit before fopen, and save its return + * value to be used as a parameter of this method. + * @param inheritData Data returned by PrefopenNoInherit + * @return Number of files processed (should be >= 1) + */ + static int PostfopenNoInherit (void *inheritData) + { + (void) inheritData; + return 1; + } + + enum Constants + { + NOTFOUND_EXITCODE = 65533, /* Process starter out of sync */ + DETACH_EXITCODE = 65534, /* Process detached */ + KILL_EXITCODE = 65535, /* Process killed or failed */ + CONSTEND + }; + + /** + * Process priorities. + */ + enum Priority + { + PRIORITY_LOWEST = -2, + PRIORITY_BELOW_NORMAL = -1, + PRIORITY_NORMAL = 0, + PRIORITY_ABOVE_NORMAL = 1, + PRIORITY_HIGHEST = 2 + }; + + /** + * Constructor. Does not start the process, use @ref Create or + * @ref CreateWithShell to actually start the process. + * @param cmdLine Command line + * @param pipeStdin set to true in order to redirect stdin + * @param stdoutListener non-NULL to redirect stdout + * @param stderrListener non-NULL to redirect stderr + * @param bufferSize Size of redirect buffers + */ + FastOS_ProcessInterface (const char *cmdLine, + bool pipeStdin = false, + FastOS_ProcessRedirectListener *stdoutListener = NULL, + FastOS_ProcessRedirectListener *stderrListener = NULL, + int bufferSize = 65535) : + _extradoublehackforalignment(0.0), + _cmdLine(NULL), + _pipeStdin(pipeStdin), + _stdoutListener(stdoutListener), + _stderrListener(stderrListener), + _bufferSize(bufferSize), + _next(NULL), + _prev(NULL) + { + _cmdLine = strdup(cmdLine); + } + + /** + * Destructor. + * If @ref Wait has not been called yet, it is called here. + */ + virtual ~FastOS_ProcessInterface () + { + free (_cmdLine); + }; + + /** + * Create and start the process. If your command line includes + * commands specific to the shell use @ref CreateWithShell instead. + * + * IPC communication currently only supports direct parent/child + * relationships. If you launch a FastOS application through + * the shell or some other script/process, the FastOS application + * might not be a direct child of your process and IPC communication + * will not work (the rest will work ok, though). + * + * This limitation might be removed in the future. + * @return Boolean success / failure + */ + virtual bool Create() = 0; + + /** + * Create and start the process using the default OS shell + * (UNIX: /bin/sh). + * + * IPC communication currently only supports direct parent/child + * relationships. If you launch a FastOS application through + * the shell or some other script/process, the FastOS application + * might not be a direct child of your process and IPC communication + * will not work (the rest will work ok, though). + * + * This limitation might be removed in the future. + * @return Boolean success / failure + */ + virtual bool CreateWithShell() = 0; + + /** + * If you are redirecting the standard input stream of the process, + * use this method to write data. To close the input stream, + * invoke @ref WriteStdin with data=NULL. If the input stream + * is not redirected, @ref WriteStdin will fail. + * @param data Pointer to data + * @param length Length of data block in bytes + * @return Boolean success / failure + */ + virtual bool WriteStdin (const void *data, size_t length) = 0; + + /** + * Terminate the process. !!IMPORTANT LIMITATION!!: There is no guarantee + * that child processes (of the process to be killed) will be killed + * as well. + * @return Boolean success / failure + */ + virtual bool Kill () = 0; + + /** + * Special case kill used in conjunction with wrapper processes + * that use process groups to kill all descendant processes. + * On UNIX, this sends SIGTERM. + * Only use this method with wrapper processes. + * @return Boolean success / failure + */ + virtual bool WrapperKill () = 0; + + /** + * Wait for the process to finish / terminate. This is called + * automatically by the destructor, but it is recommended that + * it is called as early as possible to free up resources. + * @param returnCode Pointer to int which will receive + * the process return code. + * @param timeOutSeconds Number of seconds to wait before + * the process is violently killed. + * -1 = infinite wait / no timeout + * @return Boolean success / failure + */ + virtual bool Wait (int *returnCode, int timeOutSeconds = -1) = 0; + + /** + * Poll version of @ref Wait. + * This is basically @ref Wait with a timeout of 0 seconds. + * The process is not killed if the "timeout" expires. + * A boolean value, stillRunning, is set to indicate whether + * the process is still running or not. + * There is no need to invoke @ref Wait if @ref PollWait + * indicates that the process is finished. + * @param returnCode Pointer to int which will receive + * the process return code. + * @param stillRunning Pointer to boolean value which will + * be set to indicate whether the + * process is still running or not. + * @return Boolean success / failure + */ + virtual bool PollWait (int *returnCode, bool *stillRunning) = 0; + + /** + * Detach a process, allowing it to exist beyond parent. + * Only implemented on UNIX platforms. + * + * @return Boolean success / failure + */ + virtual bool Detach(void) { return false; } + + /** + * Check if child is a direct child or a proxied child. + * + * @return Boolean true = direct, false = proxied + */ + virtual bool GetDirectChild(void) const { return true; } + + /** + * Specify that child should be a direct child. + * + * @return Boolean success / failure + */ + virtual bool SetDirectChild(void) { return true; } + + /** + * Check if child inherits open file descriptors if it's a direct child. + * + * @return Boolean true = inherit, false = close + */ + virtual bool GetKeepOpenFilesIfDirectChild(void) const { return false; } + + /** + * Specify that child should inherit open file descriptors from parent + * if it's a direct child. This reduces the cost of starting direct + * children. + * + * @return Boolean true = inherit, false = close + */ + virtual bool SetKeepOpenFilesIfDirectChild(void) { return false; } + + /** + * Get process identification number. + * @return Process id + */ + virtual unsigned int GetProcessId() = 0; + + /** + * Send IPC message to process. + * @param data Pointer to data + * @param length Length of data block in bytes + * @return Boolean success / failure + */ + virtual bool SendIPCMessage (const void *data, size_t length) = 0; + + /** + * Get command line string. + * @return Command line string + */ + const char *GetCommandLine () + { + return _cmdLine; + } + + /** + * Set process priority. Higher priorities than normal might + * require adminstrator privileges. PRIORITY_BELOW_NORMAL and + * PRIORITY_ABOVE_NORMAL are on some OSes mapped to + * PRIORITY_LOWEST and PRIORITY_HIGHEST, respectively. + * @return Boolean success / failure + */ + virtual bool SetPriority (Priority priority) = 0; +}; + +#include <vespa/fastos/unix_process.h> +typedef FastOS_UNIX_Process FASTOS_PREFIX(Process); + diff --git a/fastos/src/vespa/fastos/ringbuffer.h b/fastos/src/vespa/fastos/ringbuffer.h new file mode 100644 index 00000000000..b6866a56b37 --- /dev/null +++ b/fastos/src/vespa/fastos/ringbuffer.h @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +struct FastOS_RingBufferData +{ + union + { + unsigned int _messageSize; + uint8_t _buffer[1]; + }; +}; + + +class FastOS_RingBuffer +{ +private: + FastOS_RingBuffer (const FastOS_RingBuffer&); + FastOS_RingBuffer& operator=(const FastOS_RingBuffer&); + + bool _closed; + FastOS_RingBufferData *_data; + int _bufferSize; + int _dataIndex, _dataSize; + + int GetWriteIndex (int offset) + { + return (_dataIndex + _dataSize + offset) % _bufferSize; + } + + int GetReadIndex (int offset) + { + return (_dataIndex + offset) % _bufferSize; + } + + FastOS_Mutex _mutex; + +public: + void Reset () + { + _dataIndex = 0; + _dataSize = 0; + _closed = false; + } + + FastOS_RingBufferData *GetData () { return _data; } + + void RepositionDataAt0 () + { + uint8_t *src = &_data->_buffer[_dataIndex]; + uint8_t *dst = _data->_buffer; + + for(int i=0; i<_dataSize; i++) + *dst++ = *src++; + _dataIndex = 0; + } + + FastOS_RingBuffer (int bufferSize) + : _closed(false), + _data(0), + _bufferSize(bufferSize), + _dataIndex(0), + _dataSize(0), + _mutex() + { + _data = static_cast<FastOS_RingBufferData *> + (malloc(sizeof(FastOS_RingBufferData) + bufferSize)); + Reset(); + } + + ~FastOS_RingBuffer () + { + free(_data); + } + + uint8_t *GetWritePtr (int offset=0) + { + return &_data->_buffer[GetWriteIndex(offset)]; + } + + uint8_t *GetReadPtr (int offset=0) + { + return &_data->_buffer[GetReadIndex(offset)]; + } + + void Consume (int bytes) + { + _dataSize -= bytes; + _dataIndex = (_dataIndex + bytes) % _bufferSize; + } + + void Produce (int bytes) + { + _dataSize += bytes; + } + + int GetDataSize () + { + return _dataSize; + } + + int GetWriteSpace () + { + int spaceLeft = _bufferSize - _dataSize; + int continuousBufferLeft = _bufferSize - GetWriteIndex(0); + + if(continuousBufferLeft > spaceLeft) + continuousBufferLeft = spaceLeft; + + return continuousBufferLeft; + } + + int GetReadSpace () + { + int dataLeft = _dataSize; + int continuousBufferLeft = _bufferSize - _dataIndex; + if(continuousBufferLeft > dataLeft) + continuousBufferLeft = dataLeft; + return continuousBufferLeft; + } + + void Close () + { + _closed = true; + } + + bool GetCloseFlag () + { + return _closed; + } + + void Lock () + { + _mutex.Lock(); + } + + void Unlock () + { + _mutex.Unlock(); + } +}; + diff --git a/fastos/src/vespa/fastos/serversocket.cpp b/fastos/src/vespa/fastos/serversocket.cpp new file mode 100644 index 00000000000..eb4a737683a --- /dev/null +++ b/fastos/src/vespa/fastos/serversocket.cpp @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Implementation of FastOS_ServerSocket methods. + * + * @author Div, Oivind H. Danielsen + */ + +#include <vespa/fastos/serversocket.h> + + +/** + * Set the socket factory object. Usefull if you want the ServerSocket to + * create a custom typed socket on incoming connections. + * @param socketFactory Pointer to socket factory object to be used. + */ +void FastOS_ServerSocket::SetSocketFactory(FastOS_SocketFactory *socketFactory) +{ + _socketFactory = socketFactory; +} + + +FastOS_SocketInterface *FastOS_ServerSocket::CreateHandlerSocket(void) +{ + FastOS_SocketInterface *newSocket = NULL; + + if (_socketFactory != NULL) + { + newSocket = _socketFactory->CreateSocket(); + } + else + { + newSocket = new FastOS_Socket(); + } + + return newSocket; +} + + +FastOS_SocketInterface *FastOS_ServerSocket::Accept() +{ + FastOS_SocketInterface *handlerSocket = NULL; + int handlerSocketHandle; + struct sockaddr_storage clientAddress; + + FastOS_SocketLen clientAddressLength = sizeof(clientAddress); + + memset(&clientAddress, 0, sizeof(clientAddress)); + + handlerSocketHandle = accept(_socketHandle, + reinterpret_cast<struct sockaddr *>(&clientAddress), + &clientAddressLength); + + if (handlerSocketHandle >= 0) + { + handlerSocket = CreateHandlerSocket(); + + if (handlerSocket != NULL) + { + handlerSocket->SetUp(handlerSocketHandle, + reinterpret_cast<struct sockaddr *>(&clientAddress)); + } + } + + return handlerSocket; +} + +FastOS_Socket *FastOS_ServerSocket::AcceptPlain() +{ + FastOS_Socket *handlerSocket = NULL; + int handlerSocketHandle; + struct sockaddr_storage clientAddress; + + FastOS_SocketLen clientAddressLength = sizeof(clientAddress); + + memset(&clientAddress, 0, sizeof(clientAddress)); + + handlerSocketHandle = accept(_socketHandle, + reinterpret_cast<struct sockaddr *>(&clientAddress), + &clientAddressLength); + + if (handlerSocketHandle >= 0) + { + handlerSocket = new FastOS_Socket(); + + if (handlerSocket != NULL) + { + handlerSocket->SetUp(handlerSocketHandle, + reinterpret_cast<struct sockaddr *>(&clientAddress)); + } + } + + return handlerSocket; +} + + +bool FastOS_ServerSocket::Listen () +{ + bool rc=false; + bool reuseAddr = false; + + if(CreateIfNoSocketYet()) + { + if ((_address.ss_family == AF_INET && + reinterpret_cast<const sockaddr_in &>(_address).sin_port != 0) || + (_address.ss_family == AF_INET6 && + reinterpret_cast<const sockaddr_in6 &>(_address).sin6_port != 0)) + reuseAddr = true; + if (SetSoReuseAddr(reuseAddr)) + { + size_t socketAddrLen; + switch (_address.ss_family) + { + case AF_INET: + socketAddrLen = sizeof(sockaddr_in); + break; + case AF_INET6: + socketAddrLen = sizeof(sockaddr_in6); + { + int disable = 0; + ::setsockopt(_socketHandle, IPPROTO_IPV6, IPV6_V6ONLY, &disable, sizeof(disable)); + } + break; + default: + socketAddrLen = sizeof(sockaddr_storage); + } + if(bind(_socketHandle, reinterpret_cast<struct sockaddr *>(&_address), + socketAddrLen) >= 0) + { + if(listen(_socketHandle, _backLog) >= 0) + { + rc = true; + } + } + } + } + + return rc; +} diff --git a/fastos/src/vespa/fastos/serversocket.h b/fastos/src/vespa/fastos/serversocket.h new file mode 100644 index 00000000000..a675f1e7b83 --- /dev/null +++ b/fastos/src/vespa/fastos/serversocket.h @@ -0,0 +1,186 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definitons for FastOS_SocketFactory and FastOS_ServerSocket. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + + +#include <vespa/fastos/socket.h> + + +/** + * This base class is used to create Socket objects. You can supply + * a subclassed @ref FastOS_SocketFactory to an instance of + * @ref FastOS_ServerSocket, to have your own type of socket created + * by @ref FastOS_ServerSocket::Accept(). + */ +class FastOS_SocketFactory +{ +public: + /** + * Destructor. No cleanup needed for base class. + */ + virtual ~FastOS_SocketFactory(void) { } + + /** + * Create a socket object. Override this method to create + * your own subclassed socket objects. It is not allowed + * for the constructor to use the socket object, as it + * is not set up yet at this point. + */ + virtual FastOS_SocketInterface *CreateSocket() + { + return new FastOS_Socket(); + } +}; + + +/** + * This socket class provides a listening server socket that is + * able to accept connections on a specified port number. + * + * The port number and connection backlog are specified in the + * constructor * (FastOS_ServerSocket(int portnum, int backLog)). + * + * Call @ref Listen() to create and configure the socket for + * listening on the specified port number. + * + * To accept an incoming connection, call @ref Accept(). This will + * return a newly created @ref FastOS_Socket object to handle + * the new connection. If you want a different type of socket, + * specify your own @ref FastOS_SocketFactory with @ref SetSocketFactory(). + */ +class FastOS_ServerSocket : public FastOS_Socket +{ +private: + FastOS_ServerSocket(const FastOS_ServerSocket&); + FastOS_ServerSocket& operator=(const FastOS_ServerSocket&); + +protected: + /** + * The TCP port number to listen to. + */ + int _portNumber; + + /** + * Max number of connections in backlog. + */ + int _backLog; + + /** + * The socket factory to use for incoming connections. + * If this is NULL, the default action is to create an + * instance of the regular @ref FastOS_Socket. + */ + FastOS_SocketFactory *_socketFactory; + + /** + * Create socket for handling an incoming connection + * @return Returns pointer to newly created socket, or NULL on + * failure. + */ + FastOS_SocketInterface *CreateHandlerSocket(); + + bool _validAddress; + +public: + /** + * Constructor. If strict binding is used, call @ref GetValidAddressFlag() + * to check if setting the specified address was successful or not. + * @param portnum Listen on this port number. + * @param backLog Max number of connections in backlog. + * @param socketFactory See @ref SetSocketFactory(). + * @param strictBindHostName IP address or hostname for strict binding + */ + FastOS_ServerSocket (int portnum, int backLog=5, + FastOS_SocketFactory *socketFactory=NULL, + const char *strictBindHostName=NULL) + : _portNumber(portnum), + _backLog(backLog), + _socketFactory(socketFactory), + _validAddress(false) + { + setPreferIPv6(true); + _validAddress = SetAddress(_portNumber, strictBindHostName); + } + + bool GetValidAddressFlag () { return _validAddress; } + + /** + * Use this constructor to supply a pre-created, configured, + * bound and listening socket. When using this constructor, + * don't call @ref Listen(). + * @param socketHandle OS handle of supplied socket. + * @param socketFactory See @ref SetSocketFactory(). + */ + FastOS_ServerSocket(int socketHandle, + FastOS_SocketFactory *socketFactory) + : _portNumber(-1), + _backLog(-1), + _socketFactory(socketFactory), + _validAddress(false) + { + _socketHandle = socketHandle; + memset(&_address, 0, sizeof(_address)); + _validAddress = true; + } + + /** + * Create a listening socket. This involves creating an OS + * socket, setting SO_REUSEADDR(true), binding the socket and + * start to listen for incoming connections. You should + * call @ref Listen() if you have supplied a pre-created listening + * socket handle trough the constructor + * @ref FastOS_ServerSocket(int listenSocketHandle, FastOS_SocketFactory *socketFactory=NULL). + * @return Boolean success/failure + */ + bool Listen (); + + /** + * Accept incoming connections. The socket factory (if present) + * is used to create a socket instance for the new connection. + * Make sure you have a listening socket (see @ref Listen()) before + * calling @ref Accept(). + * @return Returns pointer to newly created socket object for the + * connection that was accepted, or NULL on failure. + */ + FastOS_SocketInterface *Accept (); + + /** + * Accept incoming connections. This version does not use the + * associated socket factory. + * Make sure you have a listening socket (see @ref Listen()) before + * calling @ref AcceptPlain(). + * @return Returns pointer to newly created socket object for the + * connection that was accepted, or NULL on failure. + */ + FastOS_Socket *AcceptPlain (); + + /** + * Specify your own @ref FastOS_SocketFactory for this serversocket. + * When new connections are accepted with @ref Accept, this socket + * factory will be called to create a new socket object for the + * connection. + * + * SetSocketFactory(NULL) will enable the default socket factory + * mechanism which will create regular @ref FastOS_Socket instances + * on accepted connections. + */ + void SetSocketFactory(FastOS_SocketFactory *socketFactory); + + /** + * Return the portnumber of the listing socket. + * @return Port number. + */ + int GetPortNumber () + { + return _portNumber; + } +}; + + diff --git a/fastos/src/vespa/fastos/socket.cpp b/fastos/src/vespa/fastos/socket.cpp new file mode 100644 index 00000000000..c7ec605d524 --- /dev/null +++ b/fastos/src/vespa/fastos/socket.cpp @@ -0,0 +1,340 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/socket.h> +#include <sstream> + +FastOS_SocketInterface::FastOS_SocketInterface() + : _readEventEnabled(false), + _writeEventEnabled(false), + _readPossible(false), + _writePossible(false), + _epolled(false), + _socketEvent(NULL), + _eventAttribute(NULL), + _socketEventArrayPos(-1), + _address(), + _socketHandle(-1), + _preferIPv6(false) +{ + ConstructorWork(); +} + +FastOS_SocketInterface::FastOS_SocketInterface(int socketHandle, struct sockaddr *hostAddress) + : _readEventEnabled(false), + _writeEventEnabled(false), + _readPossible(false), + _writePossible(false), + _epolled(false), + _socketEvent(NULL), + _eventAttribute(NULL), + _socketEventArrayPos(-1), + _address(), + _socketHandle(-1), + _preferIPv6(false) +{ + ConstructorWork(); + SetUp(socketHandle, hostAddress); +} + +FastOS_SocketInterface::~FastOS_SocketInterface() { } + +bool FastOS_SocketInterface::Connect() +{ + bool rc=false; + + if (CreateIfNoSocketYet()) { + switch (_address.ss_family) { + case AF_INET: + rc = (0 == connect(_socketHandle, + reinterpret_cast<struct sockaddr *>(&_address), + sizeof(sockaddr_in))); + break; + case AF_INET6: + rc = (0 == connect(_socketHandle, + reinterpret_cast<struct sockaddr *>(&_address), + sizeof(sockaddr_in6))); + break; + default: + rc = false; + } + } + + return rc; +} + +bool FastOS_SocketInterface::SetAddress (const int portNum, const char *address) +{ + bool rc = false; + memset(&_address, 0, sizeof(_address)); + + addrinfo hints; + memset(&hints, 0, sizeof(addrinfo)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_protocol = 0; + hints.ai_flags = (AI_PASSIVE | AI_NUMERICSERV | AI_ADDRCONFIG); + char service[32]; + snprintf(service, sizeof(service), "%d", portNum); + addrinfo *list = nullptr; + if (getaddrinfo(address, service, &hints, &list) == 0) { + const addrinfo *best = nullptr; + for (const addrinfo *info = list; info != nullptr; info = info->ai_next) { + if (best == nullptr) { + best = info; + } else if (_preferIPv6) { + if ((best->ai_family != AF_INET6) && (info->ai_family == AF_INET6)) { + best = info; + } + } else { + if ((best->ai_family != AF_INET) && (info->ai_family == AF_INET)) { + best = info; + } + } + } + if (best != nullptr) { + memcpy(&_address, best->ai_addr, best->ai_addrlen); + rc = true; + } + freeaddrinfo(list); + } + return rc; +} + +bool FastOS_SocketInterface::SetAddressByHostName (const int portNum, const char *hostName) +{ + return SetAddress(portNum, hostName); +} + +void FastOS_SocketInterface::SetUp(int socketHandle, struct sockaddr *hostAddress) +{ + Close(); + + _socketHandle = socketHandle; + switch (hostAddress->sa_family) { + case AF_INET: + memcpy(&_address, hostAddress, sizeof(sockaddr_in)); + break; + case AF_INET6: + memcpy(&_address, hostAddress, sizeof(sockaddr_in6)); + break; + default: + ; + } +} + +bool FastOS_SocketInterface::CreateIfNoSocketYet () +{ + if (ValidHandle()) { + return true; + } else if (_address.ss_family == AF_INET) { + _socketHandle = socket(AF_INET, SOCK_STREAM, 0); + return (_socketHandle != -1); + } else if (_address.ss_family == AF_INET6) { + _socketHandle = socket(AF_INET6, SOCK_STREAM, 0); + return (_socketHandle != -1); + } + return false; +} + +void FastOS_SocketInterface::ConstructorWork () +{ + _socketHandle = -1; + _epolled = false; + _socketEvent = NULL; + _readEventEnabled = false; + _writeEventEnabled = false; + _eventAttribute = NULL; + _socketEventArrayPos = -1; +} + +bool FastOS_SocketInterface::SetSoLinger( bool doLinger, int seconds ) +{ + bool rc=false; + + struct linger lingerTime; + lingerTime.l_onoff = doLinger ? 1 : 0; + lingerTime.l_linger = seconds; + + if (CreateIfNoSocketYet()) { + rc = (0 == setsockopt(_socketHandle, SOL_SOCKET, SO_LINGER, + reinterpret_cast<FastOS_SockOptValP>(&lingerTime), + sizeof(lingerTime))); + } + + return rc; +} + +bool +FastOS_SocketInterface::SetNoDelay(bool noDelay) +{ + bool rc = false; + int noDelayInt = noDelay ? 1 : 0; + + if (CreateIfNoSocketYet()) { + rc = (setsockopt(_socketHandle, IPPROTO_TCP, TCP_NODELAY, + reinterpret_cast<FastOS_SockOptValP>(&noDelayInt), + sizeof(noDelayInt)) == 0); + } + return rc; +} + +int FastOS_SocketInterface::GetSoError () +{ + if (!ValidHandle()) { return EINVAL; } + + // Fetch this first, as getsockopt(..SO_ERROR, resets the + // WSAGetLastError() + int lastError = FastOS_Socket::GetLastError(); + int soError = 0; + + FastOS_SocketLen soErrorLen = sizeof(soError); + + if (getsockopt(_socketHandle, SOL_SOCKET, SO_ERROR, + reinterpret_cast<FastOS_SockOptValP>(&soError), &soErrorLen) != 0) { + return lastError; + } + + if (soErrorLen != sizeof(soError)) { + return EINVAL; // 'invalid argument' + } + + return soError; +} + +bool FastOS_SocketInterface::SetSoIntOpt (int option, int value) +{ + bool rc=false; + + if (CreateIfNoSocketYet()) { + rc = (0 == setsockopt(_socketHandle, SOL_SOCKET, option, + reinterpret_cast<FastOS_SockOptValP>(&value), + sizeof(value))); + } + + return rc; +} + +bool FastOS_SocketInterface::GetSoIntOpt(int option, int &value) +{ + bool rc=false; + + if (CreateIfNoSocketYet()) { + FastOS_SocketLen len = sizeof(value); + + int retval = getsockopt(_socketHandle, SOL_SOCKET, option, + reinterpret_cast<FastOS_SockOptValP>(&value), + &len); + + if (len != sizeof(value)) { + // FIX! - What about GetLastError() in this case? + return false; + } + + rc = (0 == retval); + } + + return rc; +} + +void FastOS_SocketInterface::CleanupEvents () +{ + if (_socketEvent != NULL) { + _socketEvent->Detach(this); + assert(!_epolled); + _socketEvent = NULL; + } +} + +bool FastOS_SocketInterface::TuneTransport () +{ + if (!SetSoIntOpt(SO_KEEPALIVE, 1)) { return false; } + if (!SetSoLinger(true, 0)) { return false; } + int swin = 0; // SO_SNDBUF: buffer size for output + if (!GetSoIntOpt(SO_SNDBUF, swin)) { return false; } + if(swin < (32*1024)) { + swin = 32 * 1024; // magic numbers?!! + } + if (!SetSoIntOpt(SO_SNDBUF, swin)) { return false; } + return true; +} + +int FastOS_SocketInterface::GetLocalPort () +{ + int result = -1; + sockaddr_storage addr; + FastOS_SocketLen len = sizeof(addr); + if(getsockname(_socketHandle, reinterpret_cast<struct sockaddr *>(&addr), &len) == 0) { + if ((addr.ss_family == AF_INET) && (len == sizeof(sockaddr_in))) { + const sockaddr_in *my_addr = reinterpret_cast<const sockaddr_in *>(&addr); + result = ntohs(my_addr->sin_port); + } + if ((addr.ss_family == AF_INET6) && (len == sizeof(sockaddr_in6))) { + const sockaddr_in6 *my_addr = reinterpret_cast<const sockaddr_in6 *>(&addr); + result = ntohs(my_addr->sin6_port); + } + } + return result; +} + +std::string +FastOS_SocketInterface::getLastErrorString(void) { + return FastOS_Socket::getErrorString(FastOS_Socket::GetLastError()); +} + +const char * +FastOS_SocketInterface::InitializeServices(void) { + FastOS_SocketEventObjects::InitializeClass(); + return NULL; +} + +void FastOS_SocketInterface::CleanupServices () { + FastOS_SocketEventObjects::ClassCleanup(); +} + +bool FastOS_SocketInterface::SetSocketEvent (FastOS_SocketEvent *event, void *attribute) { + bool rc=false; + + _eventAttribute = attribute; + + if (CreateIfNoSocketYet()) { + if (_socketEvent != event) { + if (_socketEvent != NULL) { + // Disable events for this socket on the old SocketEvent + _socketEvent->Detach(this); + assert(!_epolled); + _socketEvent = NULL; + } + + if (event != NULL) { + event->Attach(this, _readEventEnabled, _writeEventEnabled); + _socketEvent = event; + } + } + rc = true; + } + + return rc; +} + +void FastOS_SocketInterface::EnableReadEvent (bool enabled) { + if (_readEventEnabled == enabled) { return; } + _readEventEnabled = enabled; + if (_socketEvent != NULL) { + _socketEvent->EnableEvent(this, _readEventEnabled, _writeEventEnabled); + } +} + +/** + * Enable or disable write events for the socket. + * The behaviour caused by invoking this method while waiting for + * socket events is undefined. + * A @ref FastOS_SocketEvent must be associated with the socket prior + * to calling @ref EnableReadEvent and @ref EnableWriteEvent. + */ +void FastOS_SocketInterface::EnableWriteEvent (bool enabled) { + if (_writeEventEnabled == enabled) { return; } + _writeEventEnabled = enabled; + if (_socketEvent != NULL) { + _socketEvent->EnableEvent(this, _readEventEnabled, _writeEventEnabled); + } +} diff --git a/fastos/src/vespa/fastos/socket.h b/fastos/src/vespa/fastos/socket.h new file mode 100644 index 00000000000..e7c12c5213c --- /dev/null +++ b/fastos/src/vespa/fastos/socket.h @@ -0,0 +1,316 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/fastos/types.h> + +#include <string> +#include <vector> + +class FastOS_SocketInterface; +class FastOS_ServerSocket; +class FastOS_SocketEvent; + +#include <vespa/fastos/socketevent.h> + +/** + * This class implements TCP/IP network socket. + */ +class FastOS_SocketInterface +{ +private: + friend class FastOS_SocketEvent; + +protected: + bool _readEventEnabled; + bool _writeEventEnabled; + bool _readPossible; + bool _writePossible; + bool _epolled; // true -> part of epoll set + FastOS_SocketEvent *_socketEvent; + void *_eventAttribute; + int _socketEventArrayPos; + struct sockaddr_storage _address; + int _socketHandle; + bool _preferIPv6; + + /** + * Cancel all socket events and detach from the current socket + * event, if any. + */ + void CleanupEvents (); + + /** + * Is a valid socket handle associated with this instance? + * @return True if the handle is valid, else false. + */ + bool ValidHandle () const { return (_socketHandle != -1); } + + /** + * If no OS socket is yet associated with this instance, + * create one. + * @return Boolean success/failure + */ + bool CreateIfNoSocketYet(); + + void ConstructorWork(); + +public: + FastOS_SocketInterface (const FastOS_SocketInterface&) = delete; + FastOS_SocketInterface& operator=(const FastOS_SocketInterface&) = delete; + // Static members + + /** + * Convenience method. Does GetErrorString(GetLastError()) + * @return Error string + */ + static std::string getLastErrorString(void); + + static const char *InitializeServices (); + static void CleanupServices (); + + /** + * Default constructor. Use @ref SetUp() or the various SetAddress..() + * methods to complete the socket setup. + */ + FastOS_SocketInterface(); + /** + * This constructor does @ref SetUp() with the specified parameters. + * @param socketHandle OS handle to already created socket + * @param hostAddress IP address or host name of remote host + */ + FastOS_SocketInterface(int socketHandle, struct sockaddr *hostAddress); + + /** + * Destructor + * The socket will be closed unless it was supplied by the caller. + */ + virtual ~FastOS_SocketInterface(); + + /** + * Setup of socket parameters from explicit address and socket handle + * If a socket is already associated with this instance it will be + * closed (see @ref Close()). + * The socket is closed when the socket instance is deleted. + * @param socketHandle OS handle to already created socket + * @param hostAddress Address of remote host + */ + void SetUp(int socketHandle, struct sockaddr *hostAddress); + + /** + * Set address of host to connect to. + * @param portNum IP port number of remote host + * @param hostAddress IP address or host name of remote host + * @return Boolean success/failure + */ + bool SetAddress(const int portNum, const char *hostAddress); + + /** + * Set address of host to connect to + * @param portNum IP port number of remote host + * @param hostName Hostname of remote host + * @return Boolean success/failure + */ + bool SetAddressByHostName (const int portNum, const char *hostName); + + /** + * Connects to the host using the IP Address and port number predefined. + * @return Boolean success/failure + */ + bool Connect(); + + /** + * Connects to the host using the IP Address and port number specified. + * @return Boolean success/failure + */ + bool Connect(const char *hostNameOrIPaddress, const int portNum) { + return SetAddress(portNum, hostNameOrIPaddress) ? Connect() : false; + } + + /** + * Attempts to retrieve the local port number for the socket. + * The socket must be connected when this method is called. + * @return Local port number or -1 on error. + */ + int GetLocalPort (); + + /** + * Read [buffersize] bytes to [readbuffer]. The socket must + * have a connection (see @ref Connect()) before a read is + * attempted. + * @param readBuffer Pointer to target memory buffer + * @param bufferSize Size of the buffer / bytes to read + * @return Returns number of bytes read. + */ + virtual ssize_t Read (void *readBuffer, size_t bufferSize) = 0; + + /** + * Write [buffersize] bytes from [readbuffer]. The socket must + * have a connection (see @ref Connect()) before a write is + * attempted. + * @param writeBuffer Pointer to target memory buffer + * @param bufferSize Size of the buffer / bytes to write + * @return Returns number of bytes written. + */ + virtual ssize_t Write (const void *writeBuffer, size_t bufferSize) = 0; + + /** + * Close the socket. + * Only socket handles created within this class are closed. This + * means user-supplied socket handles (through constructor or + * @ref SetUp) will not be closed. + * If the socket is already closed, the method call will return + * success. + * @return Boolean success/failure + */ + virtual bool Close () = 0; + + /** + * Shut down a connection. + * If the socket is already closed, the method call will return + * success. Socket write events are disabled. + * @return Boolean success/failure + */ + virtual bool Shutdown() = 0; + + /** + * Get socket error + * If getting the socket error fails, the error code of GetLastError() + * is returned instad. If getting the socket error succeds with a wrong + * parameter EINVAL is returned. + * @return Socket error + */ + int GetSoError(); + + /** + * Set SO_KEEPALIVE flag on socket + * @param keep SO_KEEPALIVE on (true) / off (false) + * @return Boolean success/failure + */ + bool SetSoKeepAlive (bool keep) { + return SetSoIntOpt(SO_KEEPALIVE, keep ? 1 : 0); + } + + /** + * Set SO_REUSEADDR flag on socket + * @param reuse SO_REUSEADDR on (true) / off (false) + * @return Boolean success/failure + */ + bool SetSoReuseAddr (bool reuse) { + return SetSoIntOpt(SO_REUSEADDR, reuse ? 1 : 0); + } + + /** + * Set SO_LINGER flag on socket + * @param doLinger SO_LINGER on (true) / off (false) + * @param seconds Seconds to linger after close + * @return Boolean success/failure + */ + bool SetSoLinger (bool doLinger, int seconds); + + /** + * Set TCP Nodelay option on socket + * @param noDelay Don't delay data (true) / delay data (false) + * @return Boolean success/failure + */ + bool SetNoDelay(bool noDelay); + /** + * Set socket option + * @param option Number of option to set + * @param value Value of option to set + * @return Boolean success/failure + */ + bool SetSoIntOpt (int option, int value); + + /** + * Get socket option + * @param option Number of option to get + * @param value Ref to variable for holding the value of option + * @return Boolean success/failure + */ + bool GetSoIntOpt (int option, int &value); + + /** + * Set blocking flag on socket + * @param blockingEnabled SO_BLOCKING on (true) / off (false) + * @return Boolean success/failure + */ + virtual bool SetSoBlocking (bool blockingEnabled)=0; + + /** + * Associate a socket event object with this socket. + * Any events registered with an already associated event + * object is cancelled. + * @param event Socket event object + * @param attribute Event attribute pointer + * @return Boolean success/failure. + */ + bool SetSocketEvent (FastOS_SocketEvent *event, void *attribute=NULL); + + /** + * Get socket event object + * @return Associated socket event object or NULL + */ + FastOS_SocketEvent *GetSocketEvent () { return _socketEvent; } + + /** + * Enable or disable read events for the socket. + * The behaviour caused by invoking this method while waiting for + * socket events is undefined. + * A @ref FastOS_SocketEvent must be associated with the socket prior + * to calling @ref EnableReadEvent and @ref EnableWriteEvent. + */ + void EnableReadEvent (bool enabled); + + /** + * Enable or disable write events for the socket. + * The behaviour caused by invoking this method while waiting for + * socket events is undefined. + * A @ref FastOS_SocketEvent must be associated with the socket prior + * to calling @ref EnableReadEvent and @ref EnableWriteEvent. + */ + void EnableWriteEvent (bool enabled); + + /** + * Is the socket open? + * @return True if opened, false if cloed. + */ + bool IsOpened () const { + return ValidHandle(); + } + + /** + * Return socket port. + */ + unsigned short GetPort () const + { + switch (_address.ss_family) { + case AF_INET: + return reinterpret_cast<const sockaddr_in &>(_address).sin_port; + case AF_INET6: + return reinterpret_cast<const sockaddr_in6 &>(_address).sin6_port; + default: + return 0; + } + } + + /** + * Tune the socket for transport use. + * This includes: + * SO_KEEPALIVE = 1 + * SO_LINGER = 0 + * SO_SNDBUF >= 32768 + * SO_SNDLOWAT >= 16384 + * + * @return Boolean success/failure + */ + bool TuneTransport (); + bool getPreferIPv6(void) const { return _preferIPv6; } + void setPreferIPv6(bool preferIPv6) { _preferIPv6 = preferIPv6; } +}; + +#include <vespa/fastos/unix_socket.h> +typedef FastOS_UNIX_Socket FASTOS_PREFIX(Socket); + + + diff --git a/fastos/src/vespa/fastos/socketevent.cpp b/fastos/src/vespa/fastos/socketevent.cpp new file mode 100644 index 00000000000..68cf2ce271b --- /dev/null +++ b/fastos/src/vespa/fastos/socketevent.cpp @@ -0,0 +1,315 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/socketevent.h> +#include <vespa/fastos/socket.h> + +FastOS_SocketEventObjects *FastOS_SocketEventObjects::_objects = NULL; +FastOS_Mutex FastOS_SocketEventObjects::_listMutex; +int FastOS_SocketEventObjects::_objectCount = 0; +bool FastOS_SocketEventObjects::_initialized = false; + +FastOS_SocketEvent::FastOS_SocketEvent () : + _epollfd(-1), + _epollEvents(), + _socketsInArray(0), + _getEventsIndex(0), + _wokeUp(false), + _objs(NULL) +{ + + _objs = FastOS_SocketEventObjects::ObtainObject(this); + + if(_objs != NULL) { + if(_objs->_initOk) { + } + } + epollInit(); // must be done after obtaining _objs +} + +FastOS_SocketEvent::~FastOS_SocketEvent () +{ + // Clean out potential remaining wakeup events. + bool error; + Wait(error, 0); + + epollFini(); // must be done before releasing _objs + FastOS_SocketEventObjects::ReleaseObject(_objs); +} + +bool FastOS_SocketEvent::HandleWakeUp () +{ + int wakeUpReadHandle = _objs->_wakeUpPipe[0]; + + const int MAX_WAKEUP_PIPE_READ = 128; + char dummyBytes[MAX_WAKEUP_PIPE_READ]; + + ssize_t readCount = read(wakeUpReadHandle, dummyBytes, MAX_WAKEUP_PIPE_READ); + (void) readCount; + _wokeUp = true; + return true; +} + +FastOS_SocketEventObjects *FastOS_SocketEventObjects::ObtainObject (FastOS_SocketEvent *event) +{ + FastOS_SocketEventObjects *node; + _listMutex.Lock(); + + if(_objects == NULL) + { + _objectCount++; + _listMutex.Unlock(); + + node = new FastOS_SocketEventObjects(event); + node->_next = NULL; + } + else + { + node = _objects; + _objects = node->_next; + node->_next = NULL; + + _listMutex.Unlock(); + } + + return node; +} + +void FastOS_SocketEventObjects::ReleaseObject (FastOS_SocketEventObjects *node) +{ + if (node != NULL) + node->ReleasedCleanup(); + _listMutex.Lock(); + + if (_initialized) { + node->_next = _objects; + _objects = node; + } else { + delete node; + _objectCount--; + } + + _listMutex.Unlock(); +} + + +bool +FastOS_SocketEvent::epollInit() +{ + _epollfd = epoll_create(4093); + if (_epollfd != -1 && _objs != NULL && _objs->_initOk) { + epoll_event evt; + evt.events = EPOLLIN; + evt.data.ptr = 0; + if (epoll_ctl(_epollfd, EPOLL_CTL_ADD, _objs->_wakeUpPipe[0], &evt) == 0) { + return true; // SUCCESS + } + } + epollFini(); + return false; +} + +bool +FastOS_SocketEvent::epollEnableEvent(FastOS_SocketInterface *sock, + bool read, bool write) +{ + int res = 0; + epoll_event evt; + evt.events = (read ? static_cast<uint32_t>(EPOLLIN) : 0) | (write ? static_cast<uint32_t>(EPOLLOUT) : 0); + evt.data.ptr = (void *) sock; + if (sock->_epolled) { + if (evt.events != 0) { // modify + res = epoll_ctl(_epollfd, EPOLL_CTL_MOD, sock->_socketHandle, &evt); + } else { // remove + // NB: old versions of epoll_ctl needs evt also for remove + res = epoll_ctl(_epollfd, EPOLL_CTL_DEL, sock->_socketHandle, &evt); + sock->_epolled = false; + } + } else { + if (evt.events != 0) { // add + res = epoll_ctl(_epollfd, EPOLL_CTL_ADD, sock->_socketHandle, &evt); + sock->_epolled = true; + } + } + if (res == -1) { + perror("epollEnableEvent"); + return false; + } + return true; +} + +bool +FastOS_SocketEvent::epollWait(bool &error, int msTimeout) +{ + _wokeUp = false; + int maxEvents = 256; + if ((int)_epollEvents.size() < maxEvents) { + _epollEvents.resize(maxEvents); + } + int res = epoll_wait(_epollfd, &_epollEvents[0], maxEvents, msTimeout); + error = (res == -1); + for (int i = 0; i < res; ++i) { + const epoll_event &evt = _epollEvents[i]; + FastOS_SocketInterface *sock = (FastOS_SocketInterface *) evt.data.ptr; + if (sock == NULL) { + HandleWakeUp(); + } else { + sock->_readPossible = sock->_readEventEnabled && + ((evt.events & (EPOLLIN | EPOLLERR | EPOLLHUP)) != 0); + sock->_writePossible = sock->_writeEventEnabled && + ((evt.events & (EPOLLOUT | EPOLLERR | EPOLLHUP)) != 0); + } + } + return (res > 0); +} + +int +FastOS_SocketEvent::epollGetEvents(bool *wakeUp, int msTimeout, + FastOS_IOEvent *events, int maxEvents) +{ + _wokeUp = false; + if ((int)_epollEvents.size() < maxEvents) { + _epollEvents.resize(maxEvents); + } + int res = epoll_wait(_epollfd, &_epollEvents[0], maxEvents, msTimeout); + if (res <= 0) { + return res; + } + int idx = 0; // application event index + for (int i = 0; i < res; ++i) { + const epoll_event &evt = _epollEvents[i]; + FastOS_IOEvent &appEvt = events[idx]; + FastOS_SocketInterface *sock = (FastOS_SocketInterface *) evt.data.ptr; + if (sock == NULL) { + HandleWakeUp(); // sets _wokeUp + } else { + appEvt._readOccurred = sock->_readEventEnabled && + ((evt.events & (EPOLLIN | EPOLLERR | EPOLLHUP)) != 0); + appEvt._writeOccurred = sock->_writeEventEnabled && + ((evt.events & (EPOLLOUT | EPOLLERR | EPOLLHUP)) != 0); + appEvt._eventAttribute = sock->_eventAttribute; + ++idx; + } + } + *wakeUp = _wokeUp; + return idx; +} + +void +FastOS_SocketEvent::epollFini() +{ + if (_epollfd != -1) { + // do we need to unregister pipe read before closing? + int res = close(_epollfd); + if (res == -1) { + perror("epollFini"); + } + _epollfd = -1; + } +} + +void +FastOS_SocketEventObjects::InitializeClass(void) +{ + _listMutex.Lock(); + _initialized = true; + _listMutex.Unlock(); +} + + +void FastOS_SocketEventObjects::ClassCleanup(void) +{ + _listMutex.Lock(); + _initialized = false; + for (;;) + { + FastOS_SocketEventObjects *node = _objects; + + if(node == NULL) + break; + else + { + _objects = node->_next; + delete node; + _objectCount--; + } + } + _listMutex.Unlock(); +} + + +FastOS_SocketEventObjects::FastOS_SocketEventObjects(FastOS_SocketEvent *event) + : _next(NULL), + _initOk(false), + _socketArray(NULL), + _socketArrayAllocSize(0u), + _pollfds(NULL), + _pollfdsAllocSize(0) +{ + // Connect ourselves to the socketevent object. + event->_objs = this; + + _initOk = Init(event); +} + +void +FastOS_SocketEventObjects::ReleasedCleanup(void) +{ + if (_socketArrayAllocSize > 16) { + delete [] _socketArray; + _socketArray = NULL; + _socketArrayAllocSize = 0; + } + if (_pollfdsAllocSize > 16) { + free(_pollfds); + _pollfds = NULL; + _pollfdsAllocSize = 0; + } +} + + +FastOS_SocketEventObjects::~FastOS_SocketEventObjects () +{ + Cleanup(); + delete [] _socketArray; + free(_pollfds); +} + + +void +FastOS_SocketEvent::Attach(FastOS_SocketInterface *sock, + bool readEnabled, + bool writeEnabled) +{ + assert(!sock->_epolled); + if (readEnabled || writeEnabled) + EnableEvent(sock, readEnabled, writeEnabled); +} + + +void +FastOS_SocketEvent::Detach(FastOS_SocketInterface *sock) +{ + if (sock->_readEventEnabled || sock->_writeEventEnabled) + EnableEvent(sock, false, false); +} + +void FastOS_SocketEvent::AsyncWakeUp (void) +{ + char dummy = 'c'; + size_t writeCount = write(_objs->_wakeUpPipe[1], &dummy, 1); + (void) writeCount; +} + +bool FastOS_SocketEvent::QueryReadEvent (FastOS_SocketInterface *sock) +{ + bool ret = sock->_readPossible; + sock->_readPossible = false; + return ret; +} + +bool FastOS_SocketEvent::QueryWriteEvent (FastOS_SocketInterface *sock) +{ + bool ret = sock->_writePossible; + sock->_writePossible = false; + return ret; +} diff --git a/fastos/src/vespa/fastos/socketevent.h b/fastos/src/vespa/fastos/socketevent.h new file mode 100644 index 00000000000..0a4e7f0f8f0 --- /dev/null +++ b/fastos/src/vespa/fastos/socketevent.h @@ -0,0 +1,244 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/fastos/types.h> +#include <vespa/fastos/mutex.h> +#include <poll.h> + +#include <sys/epoll.h> +#include <vector> + +class FastOS_IOEvent +{ +public: + bool _readOccurred, _writeOccurred; + void *_eventAttribute; +}; + +class FastOS_SocketEvent; +class FastOS_SocketInterface; + +class FastOS_SocketEventObjects +{ +private: + FastOS_SocketEventObjects(const FastOS_SocketEventObjects&); + FastOS_SocketEventObjects& operator=(const FastOS_SocketEventObjects&); + + static FastOS_Mutex _listMutex; + static int _objectCount; + static bool _initialized; + + bool Init (FastOS_SocketEvent *event); + void Cleanup (); + void ReleasedCleanup(void); + +public: + FastOS_SocketEventObjects *_next; + bool _initOk; + FastOS_SocketInterface **_socketArray; + unsigned int _socketArrayAllocSize; + pollfd *_pollfds; + unsigned int _pollfdsAllocSize; + int _wakeUpPipe[2]; + + FastOS_SocketEventObjects(FastOS_SocketEvent *event); + ~FastOS_SocketEventObjects (); + static FastOS_SocketEventObjects *_objects; + static FastOS_SocketEventObjects *ObtainObject (FastOS_SocketEvent *event); + static void ReleaseObject (FastOS_SocketEventObjects *node); + static void ClassCleanup(void); + static void InitializeClass(void); +}; + + +/** + * This class is used to handle events caused by @ref FastOS_Socket + * instances. + * + * A @ref FastOS_SocketEvent can be associated with multiple sockets. + * Use @ref FastOS_Socket::EnableReadEvent() and + * @ref FastOS_Socket::EnableWriteEvent() to register for read and + * and write events. + * + * A @ref Wait() sleeps until either the specified timeout elapses, + * or one or more socket events trigger. + * + * After a @ref Wait() with return code true, @ref QueryReadEvent() + * and @ref QueryWriteEvent() is used to find the socket(s) that + * caused an event. + * + * Example: + * @code + * // Simple connection class + * class Connection + * { + * public: + * Connection *_next; // Single-linked list of connections + * + * FastOS_Socket *_socket; + * void HandleReadEvent (); + * void HandleWriteEvent (); + * }; + * + * void EventExample (Connection *connections) + * { + * Connection *conn; + * FastOS_SocketEvent socketEvent; + * + * // Walk through single-linked list of connections + * for(conn=connections; conn!=NULL; conn = conn->_next) + * { + * // Associate each socket with socketEvent + * conn->_socket->SetSocketEvent(&socketEvent); + * + * // Enable read event notifications + * conn->_socket->EnableReadEvent(true); + * + * // In this example, we pretend that write events are turned + * // on somewhere else. + * } + * + * for(;;) // Event handling loop (loop forever) + * { + * // Wait for events (timeout = 200ms) + * if(socketEvent.Wait(200)) + * { + * // Walk through list of connections + * for(conn=connections; conn!=NULL; conn = conn->_next) + * { + * // For each socket, check for read event + * if(socketEvent.QueryReadEvent(conn->_socket)) + * { + * conn->HandleReadEvent(); + * } + * + * // ..and write event + * if(socketEvent.QueryWriteEvent(conn->_socket)) + * { + * conn->HandleWriteEvent(); + * } + * } + * } + * else + * { + * // Timeout + * } + * } + * } + * @endcode + */ +class FastOS_SocketEvent +{ + friend class FastOS_SocketInterface; + friend class FastOS_SocketEventObjects; + +private: + FastOS_SocketEvent(const FastOS_SocketEvent&); + FastOS_SocketEvent& operator=(const FastOS_SocketEvent&); +protected: + + int _epollfd; // fd of epoll kernel service + std::vector<epoll_event> _epollEvents; // internal epoll event storage + + int _socketsInArray; + int _getEventsIndex; + + bool _wokeUp; + + FastOS_SocketEventObjects *_objs; + + bool HandleWakeUp (); + void EnableEvent (FastOS_SocketInterface *sock, bool read, bool write) { + epollEnableEvent(sock, read, write); + } + + bool epollInit(); + bool epollEnableEvent(FastOS_SocketInterface *sock, bool read, bool write); + bool epollWait(bool &error, int msTimeout); + int epollGetEvents(bool *wakeUp, int msTimeout, + FastOS_IOEvent *events, int maxEvents); + void epollFini(); + +public: + FastOS_SocketEvent (); + ~FastOS_SocketEvent (); + + /** + * Was the socketevent object created successfully? + * @return Boolean success/failure + */ + bool GetCreateSuccess () { + if (_epollfd == -1) { + return false; + } + return (_objs != NULL) ? _objs->_initOk : false; + } + + /** + * Wait for a socket event, or timeout after [msTimeout] milliseconds. + * + * @param error This will be set to true if an error occured. + * If Wait succeeds, this will always be false. + * @param msTimeout Number of milliseconds to wait for an event + * before timeout. -1 means wait forever. + * + * @return True if an event occurred, else false. + */ + bool Wait (bool &error, int msTimeout) { + return epollWait(error, msTimeout); + } + + /** + * Wait for socket event, or timeout after [msTimeout] milliseconds. + * An array of IO events is filled in. + * + * @param wakeUp Has a wakeup occurred? (out parameter) + * @param msTimeout Number of milliseconds to wait for an event + * before timeout. -1 means wait forever. + * @param events Pointer to FastOS_IOEvent array + * @param maxEvents Size of event array. Up to this many events + * may be filled in the array. Invoke the method + * multiple times to get all events if the array + * is too small to hold all events that occurred. + * + * @return Number of events occurred, or -1 on failure. + */ + int GetEvents (bool *wakeUp, int msTimeout, FastOS_IOEvent *events, int maxEvents) { + return epollGetEvents(wakeUp, msTimeout, events, maxEvents); + } + + /** + * Make FastOS_SocketEvent methods Wait/GetEvents + * stop waiting and return ASAP. + */ + void AsyncWakeUp (void); + + void Attach(FastOS_SocketInterface *sock, bool read, bool write); + + void Detach(FastOS_SocketInterface *sock); + + /** + * Check for a read-event with socket [socket]. + * + * This method will also clear the read event indication for the + * given socket, making this method return false for the given + * socket until another event has been detected by invoking + * Wait/GetEvents. + * + * @return True if an event occurred, else false. + */ + bool QueryReadEvent (FastOS_SocketInterface *socket); + + /** + * Check for a write-event with socket [socket]. + * + * This method will also clear the write event indication for the + * given socket, making this method return false for the given + * socket until another event has been detected by invoking + * Wait/GetEvents. + * + * @return True if an event occurred, else false. + */ + bool QueryWriteEvent (FastOS_SocketInterface *socket); +}; diff --git a/fastos/src/vespa/fastos/thread.cpp b/fastos/src/vespa/fastos/thread.cpp new file mode 100644 index 00000000000..d72939699cb --- /dev/null +++ b/fastos/src/vespa/fastos/thread.cpp @@ -0,0 +1,373 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Implementation of FastOS_ThreadPool and FastOS_Thread methods. + * + * @author Oivind H. Danielsen + */ + +#include <vespa/fastos/thread.h> + +// ---------------------------------------------------------------------- +// FastOS_ThreadPool +// ---------------------------------------------------------------------- + +FastOS_ThreadPool::FastOS_ThreadPool(int stackSize, int maxThreads) + : _startedThreadsCount(0), + _closeFlagMutex(), + _stackSize(stackSize), + _closeCalledFlag(false), + _freeMutex(), + _liveCond(), + _freeThreads(NULL), + _activeThreads(NULL), + _numFree(0), + _numActive(0), + _numTerminated(0), + _numLive(0), + _maxThreads(maxThreads) // 0 means unbounded +{ +} + +FastOS_ThreadPool::~FastOS_ThreadPool(void) +{ + Close(); +} + +void FastOS_ThreadPool::ThreadIsAboutToTerminate(FastOS_ThreadInterface *) +{ + assert(isClosed()); + + _liveCond.Lock(); + + _numTerminated++; + _numLive--; + if (_numLive == 0) + _liveCond.Broadcast(); + + _liveCond.Unlock(); +} + + +// This is a NOP if the thread isn't active. +void FastOS_ThreadPool::FreeThread (FastOS_ThreadInterface *thread) +{ + _freeMutex.Lock(); + + if(thread->_active) { + LinkOutThread(thread, &_activeThreads); + + thread->_active = false; + _numActive--; + + LinkInThread(thread, &_freeThreads); + _numFree++; + } + + _freeMutex.Unlock(); +} + +void FastOS_ThreadPool::LinkOutThread (FastOS_ThreadInterface *thread, FastOS_ThreadInterface **listHead) +{ + if (thread->_prev != NULL) + thread->_prev->_next = thread->_next; + if (thread->_next != NULL) + thread->_next->_prev = thread->_prev; + + if (thread == *listHead) + *listHead = thread->_next; +} + +void FastOS_ThreadPool::LinkInThread (FastOS_ThreadInterface *thread, FastOS_ThreadInterface **listHead) +{ + thread->_prev = NULL; + thread->_next = *listHead; + + if (*listHead != NULL) + (*listHead)->_prev = thread; + + *listHead = thread; +} + + +// _freeMutex is held by caller. +void FastOS_ThreadPool::ActivateThread (FastOS_ThreadInterface *thread) +{ + LinkOutThread(thread, &_freeThreads); + LinkInThread(thread, &_activeThreads); + + thread->_active = true; + _numActive++; + _startedThreadsCount++; +} + + +// Allocate a thread, either from pool of free or by 'new'. Finally, +// make this thread call parameter fcn when it becomes active. +FastOS_ThreadInterface *FastOS_ThreadPool::NewThread (FastOS_Runnable *owner, void *arg) +{ + FastOS_ThreadInterface *thread=NULL; + + _freeMutex.Lock(); + + if (!isClosed()) { + if ((thread = _freeThreads) != NULL) { + // Reusing thread entry + _freeThreads = thread->_next; + _numFree--; + + ActivateThread(thread); + } else { + // Creating new thread entry + + if (_maxThreads != 0 && ((_numActive + _numFree) >= _maxThreads)) { + fprintf(stderr, "Error: Maximum number of threads (%d)" + " already allocated.\n", _maxThreads); + } else { + _freeMutex.Unlock(); + + _liveCond.Lock(); + _numLive++; + _liveCond.Unlock(); + + thread = FastOS_Thread::CreateThread(this); + + if (thread == NULL) { + _liveCond.Lock(); + _numLive--; + if (_numLive == 0) { + _liveCond.Broadcast(); + } + _liveCond.Unlock(); + } + + _freeMutex.Lock(); + + if(thread != NULL) + ActivateThread(thread); + } + } + } + + _freeMutex.Unlock(); + if(thread != NULL) { + _liveCond.Lock(); + thread->Dispatch(owner, arg); + _liveCond.Unlock(); + } + + return thread; +} + + +void FastOS_ThreadPool::BreakThreads () +{ + FastOS_ThreadInterface *thread; + + _freeMutex.Lock(); + + // Notice all active threads that they should quit + for(thread=_activeThreads; thread != NULL; thread=thread->_next) { + thread->SetBreakFlag(); + } + + // Notice all free threads that they should quit + for(thread=_freeThreads; thread != NULL; thread=thread->_next) { + thread->SetBreakFlag(); + } + + _freeMutex.Unlock(); +} + + +void FastOS_ThreadPool::JoinThreads () +{ + _liveCond.Lock(); + + while (_numLive > 0) + _liveCond.Wait(); + + _liveCond.Unlock(); +} + +void FastOS_ThreadPool::DeleteThreads () +{ + FastOS_ThreadInterface *thread; + + _freeMutex.Lock(); + + assert(_numActive == 0); + assert(_numLive == 0); + + while((thread = _freeThreads) != NULL) { + LinkOutThread(thread, &_freeThreads); + _numFree--; + // printf("deleting thread %p\n", thread); + delete(thread); + } + + assert(_numFree == 0); + + _freeMutex.Unlock(); +} + +void FastOS_ThreadPool::Close () +{ + _closeFlagMutex.Lock(); + if (!_closeCalledFlag) { + _closeCalledFlag = true; + _closeFlagMutex.Unlock(); + + BreakThreads(); + JoinThreads(); + DeleteThreads(); + } else { + _closeFlagMutex.Unlock(); + } +} + +bool FastOS_ThreadPool::isClosed() +{ + _closeFlagMutex.Lock(); + bool closed(_closeCalledFlag); + _closeFlagMutex.Unlock(); + return closed; +} + +extern "C" +{ +void *FastOS_ThreadHook (void *arg) +{ + FastOS_ThreadInterface *thread = static_cast<FastOS_ThreadInterface *>(arg); + thread->Hook(); + + return NULL; +} +}; + + +// ---------------------------------------------------------------------- +// FastOS_ThreadInterface +// ---------------------------------------------------------------------- + +void FastOS_ThreadInterface::Hook () +{ + // Loop forever doing the following: Wait on the signal _dispatched. + // When awoken, call _start_fcn with the parameters. Then zero set + // things and return this to the owner, i.e. pool of free threads + bool finished=false; + bool deleteOnCompletion = false; + + while(!finished) { + + _dispatched.Lock(); // BEGIN lock + + while (_owner == NULL && !(finished = _pool->isClosed())) { + _dispatched.Wait(); + } + + _dispatched.Unlock(); // END lock + + if(!finished) { + PreEntry(); + deleteOnCompletion = _owner->DeleteOnCompletion(); + _owner->Run(this, _startArg); + + _dispatched.Lock(); // BEGIN lock + + if (deleteOnCompletion) { + delete _owner; + } + _owner = NULL; + _startArg = NULL; + _breakFlag = false; + finished = _pool->isClosed(); + + _dispatched.Unlock(); // END lock + + _runningCond.ClearBusyBroadcast(); + + _pool->FreeThread(this); + // printf("Thread given back to FastOS_ThreadPool: %p\n", this); + } + } + + _pool->ThreadIsAboutToTerminate(this); + + // Be sure not to touch any members from here on, as we are about + // to be deleted. +} + + +// Make this thread call parameter fcn with parameters argh +// when this becomes active. +// Restriction: _liveCond must be held by the caller. + +void FastOS_ThreadInterface::Dispatch(FastOS_Runnable *newOwner, void *arg) +{ + _dispatched.Lock(); + + _runningCond.SetBusy(); + + _owner = newOwner; + _startArg = arg; + // Set _thread variable before NewThread returns + _owner->_thread = this; + + // It is safe to signal after the unlock since _liveCond is still held + // so the signalled thread still exists. + // However as thread creation is infrequent and as helgrind suggest doing + // it the safe way we just do that, instead of keeping a unneccessary long + // suppressionslist. It will be long enough anyway. + + _dispatched.Signal(); + + _dispatched.Unlock(); +} + +void FastOS_ThreadInterface::SetBreakFlag() +{ + _dispatched.Lock(); + _breakFlag = true; + + _dispatched.Signal(); + _dispatched.Unlock(); +} + + +FastOS_ThreadInterface *FastOS_ThreadInterface::CreateThread(FastOS_ThreadPool *pool) +{ + FastOS_ThreadInterface *thread = new FastOS_Thread(pool); + + if(!thread->Initialize(pool->GetStackSize(), pool->GetStackGuardSize())) { + delete(thread); + thread = NULL; + } + + return thread; +} + +void FastOS_ThreadInterface::Join () +{ + _runningCond.WaitBusy(); +} + + +// ---------------------------------------------------------------------- +// FastOS_Runnable +// ---------------------------------------------------------------------- + +FastOS_Runnable::FastOS_Runnable(void) + : _thread(NULL) +{ +} + +FastOS_Runnable::~FastOS_Runnable(void) +{ + // assert(_thread == NULL); +} + +void FastOS_Runnable::Detach(void) +{ + _thread = NULL; +} diff --git a/fastos/src/vespa/fastos/thread.h b/fastos/src/vespa/fastos/thread.h new file mode 100644 index 00000000000..db0baf7e3d9 --- /dev/null +++ b/fastos/src/vespa/fastos/thread.h @@ -0,0 +1,523 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definitions for FastOS_ThreadPool, FastOS_ThreadInterface and + * FastOS_Runnable. + * + * @author Oivind H. Danielsen + */ + +#pragma once + + +#include <vespa/fastos/types.h> +#include <vespa/fastos/mutex.h> +#include <vespa/fastos/cond.h> + + +class FastOS_Runnable; +class FastOS_ThreadInterface; + + +/** + * This class implements an initially empty pool of threads. + * + * As threads are allocated with @ref NewThread() the number of + * threads in the pool increases. A maximum number of threads + * contained in the pool can be set using the constructor + * FastOS_ThreadPool(int maxThreads). + * + * Threads are automatically returned to the pool when they + * terminate. + */ +class FastOS_ThreadPool +{ + friend class FastOS_ThreadInterface; + +private: + FastOS_ThreadPool(const FastOS_ThreadPool&); + FastOS_ThreadPool& operator=(const FastOS_ThreadPool&); + + int _startedThreadsCount; + FastOS_Mutex _closeFlagMutex; + /** + * The stack size for threads in this pool. + */ + const int _stackSize; + bool _closeCalledFlag; + + // Always lock in this order + FastOS_Mutex _freeMutex; + FastOS_Cond _liveCond; + /** + * List of free (available) threads. + */ + FastOS_ThreadInterface *_freeThreads; + + /** + * List of active (allocated) threads. + */ + FastOS_ThreadInterface *_activeThreads; + + /** + * Number of available threads in the threadpool. + * Total number of threads = free + active + */ + int _numFree; + + /** + * Number of active threads in the threadpool. + * Total number of threads = free + active + */ + int _numActive; + + /** + * Number of threads that have terminated + */ + int _numTerminated; + + /** + * Number of threads that have not been terminated + */ + int _numLive; + + /** + * Maximum number of threads in the threadpool. A value of + * zero means that there is no limit. + */ + int _maxThreads; + + /** + * Put this thread on the @ref _activeThreads list. + */ + void ActivateThread (FastOS_ThreadInterface *thread); + + /** + * Return previously active thread to the list of free thread. + */ + void FreeThread (FastOS_ThreadInterface *thread); + + /** + * A thread is informing the thread pool that it is about to + * terminate. + */ + void ThreadIsAboutToTerminate(FastOS_ThreadInterface *thread); + + /** + * Set the break flag on all threads. + */ + void BreakThreads (); + + /** + * Wait for all threads to finish. + */ + void JoinThreads (); + + /** + * Delete all threads in threadpool. + */ + void DeleteThreads (); + + /** + * Remove a thread from a list. + */ + void LinkOutThread (FastOS_ThreadInterface *thread, + FastOS_ThreadInterface **listHead); + + /** + * Add a thread to a list. Notice that a thread can be on only one + * list at a time. + */ + void LinkInThread (FastOS_ThreadInterface *thread, + FastOS_ThreadInterface **listHead); + +public: + /** + * Create a threadpool that can hold a maximum of [maxThreads] threads. + * @param stackSize The stack size for threads in this pool should + * be this many bytes. + * @param maxThreads Maximum number of threads in threadpool. + * (0 == no limit). + */ + FastOS_ThreadPool(int stackSize, int maxThreads=0); + + /** + * Destructor. Closes pool if necessary. + */ + virtual ~FastOS_ThreadPool(void); + + + /** + * Allocate a new thread, and make this thread invoke the Run() method + * of the @ref FastOS_Runnable object [owner] with parameters [arg]. + * The thread is automatically freed (returned to the treadpool) + * when Run() returns. + * + * @param owner Instance to be invoked by new thread. + * @param arg Arguments to be passed to new thread. + * + * @return Pointer to newly created thread or NULL on failure. + */ + FastOS_ThreadInterface *NewThread (FastOS_Runnable *owner, void *arg=NULL); + + /** + * Get the stack size used for threads in this pool. + * @return Stack size in bytes. + */ + int GetStackSize(void) const { return _stackSize; } + + int GetStackGuardSize(void) const { return 0; } + + /** + * Close the threadpool. This involves setting the break flag on + * all active threads, and waiting for them to finish. Once Close + * is called, no more threads can be allocated from the thread + * pool. There exists no way to reopen a closed threadpool. + */ + void Close (); + + /** + * This will tell if the pool has been closed. + */ + bool isClosed(); + + /** + * Get the number of currently active threads. + * The total number of actual allocated threads is the sum of + * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). + * @return Number of currently active threads + */ + int GetNumActiveThreads () const { return _numActive; } + + /** + * Get the number of currently inactive threads. + * The total number of actual allocated threads is the sum of + * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). + * @return Number of currently inactive threads + */ + int GetNumInactiveThreads () const { return _numFree; } + + /** + * Get the number of started threads since instantiation of the thread pool. + * @return Number of threads started + */ + int GetNumStartedThreads () const { return _startedThreadsCount; } +}; + + +// Operating system thread entry point +FASTOS_EXTERNC +{ + void *FastOS_ThreadHook (void *arg); +}; + +/** + * This class controls each operating system thread. + * + * In most cases you would not want to create objects of this class + * directly. Use @ref FastOS_ThreadPool::NewThread() instead. + */ +class FastOS_ThreadInterface +{ + friend class FastOS_ThreadPool; + friend void *FastOS_ThreadHook (void *arg); + +private: + FastOS_ThreadInterface(const FastOS_ThreadInterface&); + FastOS_ThreadInterface& operator=(const FastOS_ThreadInterface&); + +protected: + /** + * The thread does not start (call @ref FastOS_Runnable::Run()) + * until this event has been triggered. + */ + FastOS_Cond _dispatched; + + FastOS_ThreadInterface *_next; + FastOS_ThreadInterface *_prev; + + /** + * A pointer to the instance which implements the interface + * @ref FastOS_Runnable. + */ + FastOS_Runnable *_owner; + + /** + * A pointer to the originating @ref FastOS_ThreadPool + */ + FastOS_ThreadPool *_pool; + + /** + * Entry point for the OS thread. The thread will sleep here + * until dispatched. + */ + void Hook (); + + /** + * Signals that thread should be dispatched. + * @param owner Instance of @ref FastOS_Runnable. + * @param arg Thread invocation arguments. + */ + void Dispatch (FastOS_Runnable *owner, void *arg); + + /** + * This method is called prior to invoking @ref FastOS_Runnable::Run(). + * Usually this involves setting operating system thread attributes, + * and is handled by each operating specific subclass. + */ + virtual void PreEntry ()=0; + + /** + * Initializes a thread. This includes creating the operating system + * socket handle and setting it up and making it ready to be dispatched. + * @return Boolean success/failure + */ + virtual bool Initialize (int stackSize, int stackGuardSize)=0; + + /** + * Used to store thread invocation arguments. These are passed along + * to @ref FastOS_Runnable::Run() when the thread is dispatched. + */ + void *_startArg; + + /** + * Create an operating system thread. In most cases you would want + * to create threads using @ref FastOS_ThreadPool::NewThread() instead. + * @param pool The threadpool which is about to contain the new thread. + * @return A new @ref FastOS_Thread or NULL on failure. + */ + static FastOS_ThreadInterface *CreateThread(FastOS_ThreadPool *pool); + + /** + * Break flag. If true, the thread should exit. + */ + bool _breakFlag; + + /** + * Is this thread active or free in the threadpool? + */ + bool _active; + + /** + * Is the thread running? This is used by @ref Join(), to wait for threads + * to finish. + */ + FastOS_BoolCond _runningCond; + +public: + /** + * Initialize the thread class. This is invoked by + * @ref FastOS_Application::Init(). + * @return Boolean success/failure + */ + static bool InitializeClass () {return true;}; + + /** + * Cleanup the thread class. This is invoked by + * @ref FastOS_Application::Cleanup(). + * @return Boolean success/failure + */ + static bool CleanupClass () {return true;}; + + /** + * Constructor. Resets internal attributes. + */ + FastOS_ThreadInterface (FastOS_ThreadPool *pool) + : _dispatched(), + _next(NULL), + _prev(NULL), + _owner(NULL), + _pool(pool), + _startArg(NULL), + _breakFlag(false), + _active(false), + _runningCond() + { + } + + /** + * Destructor. + */ + virtual ~FastOS_ThreadInterface (){} + + /** + * Sleep for x milliseconds. Attempting to sleep for <1 milliseconds + * will result in failure. + * @param ms Number of milliseconds to sleep. + * @return Boolean success/failure + */ + static bool Sleep(int ms); + + /** + * Thread priorities. The range is -2..2. A higher number + * indicates higher priority. + */ + enum Priority + { + PRIORITY_LOWEST = -2, + PRIORITY_BELOW_NORMAL = -1, + PRIORITY_NORMAL = 0, + PRIORITY_ABOVE_NORMAL = 1, + PRIORITY_HIGHEST = 2 + }; + + /** + * Set thread priority. + * @param priority New thread priority. + * @return Boolean success / failure + */ + virtual bool SetPriority (const Priority priority) = 0; + + /** + * Instruct a thread to exit. This could be used in conjunction with + * @ref GetBreakFlag() in a worker thread, to have cooperative thread + * termination. When a threadpool closes, all threads in the pool will + * have their break flag set. + */ + void SetBreakFlag (); + + /** + * Return the status of this thread's break flag. If the break flag + * is set, someone wants the thread to terminate. It is up to the + * implementor of the thread to decide whether the break flag + * should be used. + * + * In scenarios where a worker thread loops "forever" waiting for + * new jobs, the break flag should be polled in order to eventually + * exit from the loop and terminate the thread. + * + * In scenarios where a worker thread performs a task which + * always should run to completion, the break flag could be ignored + * as the thread sooner or later will terminate. + * + * When a threadpool is closed, the break flag is set on all + * threads in the pool. If a thread loops forever and chooses to + * ignore the break flag, a @ref FastOS_ThreadPool::Close() will + * never finish. (see @ref SetBreakFlag) + */ + bool GetBreakFlag () const + { + return _breakFlag; + } + + /** + * Wait for a thread to finish. + */ + void Join (); + + /** + * Returns the id of this thread. + */ + virtual FastOS_ThreadId GetThreadId ()=0; +}; + + +/** + * This class gives a generic interface for invoking new threads with an object. + * + * The thread object should inherit this interface (class), and implement + * the @ref Run() method. When @ref FastOS_ThreadPool::NewThread() is + * called, the @ref Run() method of the passed instance will be invoked. + * + * Arguments could be supplied via @ref FastOS_ThreadPool::NewThread(), but + * it is also possible to supply arguments to the new thread through the + * worker thread object constructor or some other attribute-setting method + * prior to creating the thread. Choose whichever method works best for you. + * + * Example: + * @code + * // Arguments passed to the new thread. + * struct MyThreadArgs + * { + * int _something; + * char _tenChars[10]; + * }; + * + * class MyWorkerThread : public FastOS_Runnable + * { + * public: + * + * // Delete this instance upon completion + * virtual bool DeleteOnCompletion() const { return true; } + * + * virtual void Run (FastOS_ThreadInterface *thread, void *arguments) + * { + * MyThreadArgs *args = static_cast<MyThreadArgs *>(arguments); + * + * // Do some computation... + * Foo(args->_something); + * + * for(int i=0; i<30000; i++) + * { + * ... + * ... + * + * if(thread->GetBreakFlag()) + * break; + * ... + * ... + * + * } + * + * // Thread terminates... + * } + * }; + * + * + * // Example on how to create a thread using the above classes. + * void SomeClass::SomeMethod (FastOS_ThreadPool *pool) + * { + * MyWorkerThread *workerThread = new MyWorkerThread(); + * static MyThreadArgs arguments; + * + * arguments._something = 123456; + * + * // the workerThread instance will be deleted when Run completes + * // see the DeleteOnCompletion doc + * pool->NewThread(workerThread, &arguments); + * } + * @endcode + */ +class FastOS_Runnable +{ +private: + FastOS_Runnable(const FastOS_Runnable&); + FastOS_Runnable& operator=(const FastOS_Runnable&); + +protected: + friend class FastOS_ThreadInterface; + FastOS_ThreadInterface *_thread; + +public: + FastOS_Runnable(); + virtual ~FastOS_Runnable(); + + /** + * The DeleteOnCompletion method should be overridden to return true + * if the runnable instance should be deleted when run completes + * + * @author Nils Sandoy + * @return true iff this runnable instance should be deleted on completion + */ + virtual bool DeleteOnCompletion() const { return false; } + + /** + * When an object implementing interface @ref FastOS_Runnable is used to + * create a thread, starting the thread causes the object's @ref Run() + * method to be called in that separately executing thread. The thread + * terminates when @ref Run() returns. + * @param thisThread A thread object. + * @param arguments Supplied to @ref FastOS_ThreadPool::NewThread + */ + virtual void Run(FastOS_ThreadInterface *thisThread, void *arguments)=0; + + FastOS_ThreadInterface *GetThread(void) { return _thread; } + const FastOS_ThreadInterface *GetThread(void) const { return _thread; } + bool HasThread(void) const { return _thread != NULL; } + void Detach(void); +}; + +#include <vespa/fastos/unix_thread.h> +typedef FastOS_UNIX_Thread FASTOS_PREFIX(Thread); + diff --git a/fastos/src/vespa/fastos/time.cpp b/fastos/src/vespa/fastos/time.cpp new file mode 100644 index 00000000000..1a824c7132f --- /dev/null +++ b/fastos/src/vespa/fastos/time.cpp @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/time.h> + +double +FastOS_TimeInterface::MicroSecsToNow(void) const +{ + FastOS_Time now; + now.SetNow(); + return now.MicroSecs() - MicroSecs(); +} + +double +FastOS_TimeInterface::MilliSecsToNow(void) const +{ + FastOS_Time now; + now.SetNow(); + return now.MilliSecs() - MilliSecs(); +} diff --git a/fastos/src/vespa/fastos/time.h b/fastos/src/vespa/fastos/time.h new file mode 100644 index 00000000000..97ebe21a3b5 --- /dev/null +++ b/fastos/src/vespa/fastos/time.h @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/types.h> + +/** + * Interface to OS time functions. + */ +class FastOS_TimeInterface +{ +protected: + /** + * Destructor. No cleanup needed for base class. + */ + virtual ~FastOS_TimeInterface(void) { } + +public: + /** + * Set the time to 0. + */ + virtual void SetZero() = 0; + + /** + * Set the time, specified by number of seconds. + * @param seconds Number of seconds. + */ + FastOS_TimeInterface& operator=(const double seconds) + { + SetSecs(seconds); + return *this; + } + + /** + * Return the microsecond difference between the current time + * and the time stored in the instance. + * Note: Only millisecond accuracy is guaranteed. + * @return Time difference in microseconds. + */ + double MicroSecsToNow() const; + /** + * Return the millisecond difference between the current time + * and the time stored in the instance. + * @return Time difference in milliseconds. + */ + double MilliSecsToNow() const; + + /** + * Add a specified number of microseconds to the time. + * Note: Only millisecond accuracy is guaranteed. + * @param microsecs Number of microseconds to add. + */ + void AddMicroSecs(double microsecs) + { SetMicroSecs(MicroSecs() + microsecs); } + + /** + * Add a specified number of milliseconds to the time. + * @param millisecs Number of milliseconds to add. + */ + void AddMilliSecs(double millisecs) + { SetMilliSecs(MilliSecs() + millisecs); } + + /** + * Subtract a specified number of microseconds from the time. + * Note: Only millisecond accuracy is guaranteed. + * @param microsecs Number of microseconds to subtract. + */ + void SubtractMicroSecs(double microsecs) + { SetMicroSecs(MicroSecs() - microsecs); } + + /** + * Subtract a specified number of milliseconds from the time. + * @param millisecs Number of milliseconds to subtract. + */ + void SubtractMilliSecs(double millisecs) + { SetMilliSecs(MilliSecs() - millisecs); } + + /** + * Return the time in microseconds. + * Note: Only millisecond accuracy is guaranteed. + * @return Time in microseconds. + */ + virtual double MicroSecs() const = 0; + + /** + * Return the time in milliseconds. + * @return Time in milliseconds. + */ + virtual double MilliSecs() const = 0; + + /** + * Return the time in seconds. + * @return Time in seconds. + */ + virtual double Secs() const = 0; + + /** + * Set the time, specified in microseconds. + * Note: Only millisecond accuracy is guaranteed. + * @param microsecs Time in microseconds. + */ + virtual void SetMicroSecs(double microsecs) = 0; + + /** + * Set the time, specified in milliseconds. + * @param millisecs Time in milliseconds. + */ + virtual void SetMilliSecs(double millisecs) = 0; + + /** + * Set the time, specified in seconds. + * @param secs Time in seconds. + */ + virtual void SetSecs(double secs) = 0; + + /** + * Set the time value to the current system time. + */ + virtual void SetNow() = 0; + + /** + * Get the seconds-part of the time value. If the time value + * is 56.1234, this method will return 56. + * @return Number of seconds. + */ + virtual long int GetSeconds() const = 0; + + /** + * Get the microsecond-part of the time value. If the time + * value is 56.123456, this method will return 123456. + * Note: Only millisecond accuracy is guaranteed. + * @return Number of microseconds. + */ + virtual long int GetMicroSeconds() const = 0; +}; + + +#include <vespa/fastos/unix_time.h> +typedef FastOS_UNIX_Time FASTOS_PREFIX(Time); + diff --git a/fastos/src/vespa/fastos/timestamp.cpp b/fastos/src/vespa/fastos/timestamp.cpp new file mode 100644 index 00000000000..3f56a37b4e2 --- /dev/null +++ b/fastos/src/vespa/fastos/timestamp.cpp @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/timestamp.h> +#include <vespa/fastos/mutex.h> +#include <cmath> + +namespace fastos { + +const TimeStamp::TimeT TimeStamp::MILLI; +const TimeStamp::TimeT TimeStamp::MICRO; +const TimeStamp::TimeT TimeStamp::NANO; +const TimeStamp::TimeT TimeStamp::US; +const TimeStamp::TimeT TimeStamp::MS; +const TimeStamp::TimeT TimeStamp::SEC; +const TimeStamp::TimeT TimeStamp::MINUTE; + +std::string +TimeStamp::asString(double timeInSeconds) +{ + double intpart; + double fractpart = modf(timeInSeconds, &intpart); + time_t timeStamp = (time_t)intpart; + struct tm timeStruct; + gmtime_r(&timeStamp, &timeStruct); + char timeString[128]; + strftime(timeString, sizeof(timeString), "%F %T", &timeStruct); + char retval[128]; + uint32_t milliSeconds = std::min((uint32_t)(fractpart * 1000.0), 999u); + snprintf(retval, sizeof(retval), "%s.%03u UTC", timeString, milliSeconds); + return std::string(retval); +} + +int64_t ClockSystem::now() +{ + struct timeval timeNow; + gettimeofday(&timeNow, NULL); + int64_t ns = timeNow.tv_sec; + ns *= TimeStamp::NANO; + ns += timeNow.tv_usec*1000; + return ns; +} + +} diff --git a/fastos/src/vespa/fastos/timestamp.h b/fastos/src/vespa/fastos/timestamp.h new file mode 100644 index 00000000000..ee5b29e0baf --- /dev/null +++ b/fastos/src/vespa/fastos/timestamp.h @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/types.h> +#include <vespa/fastos/mutex.h> +#include <limits> +#include <string> + +namespace fastos { + +class TimeStamp +{ +public: + typedef int64_t TimeT; + static const TimeT MILLI = 1000LL; + static const TimeT MICRO = 1000*MILLI; + static const TimeT NANO = 1000*MICRO; + static const TimeT US = MILLI; + static const TimeT MS = MICRO; + static const TimeT SEC = NANO; + static const TimeT MINUTE = 60*SEC; + class Seconds { + public: + explicit Seconds(double v) : _v(v * NANO) {} + TimeT val() const { return _v; } + private: + TimeT _v; + }; + enum Special { FUTURE }; + TimeStamp() : _time(0) { } + TimeStamp(const timeval & tv) : _time(tv.tv_sec*SEC + tv.tv_usec*MILLI) { } + TimeStamp(Special s) : _time(std::numeric_limits<TimeT>::max()) { (void) s; } + TimeStamp(TimeT v) : _time(v) { } + TimeStamp(int32_t v) : _time(v) { } + TimeStamp(uint32_t v) : _time(v) { } + TimeStamp(uint64_t v) : _time(v) { } + TimeStamp(Seconds v) : _time(v.val()) { } + TimeT val() const { return _time; } + operator TimeT () const { return val(); } + TimeStamp & operator += (const TimeStamp & b) { _time += b._time; return *this; } + TimeStamp & operator -= (const TimeStamp & b) { _time -= b._time; return *this; } + TimeT time() const { return val()/NANO; } + TimeT ms() const { return val()/1000000; } + TimeT us() const { return val()/1000; } + TimeT ns() const { return val(); } + double sec() const { return val()/1000000000.0; } + std::string toString() const { return asString(sec()); } + static std::string asString(double timeInSeconds); +private: + TimeT _time; +}; + +inline TimeStamp operator +(const TimeStamp & a, const TimeStamp & b) { return TimeStamp(a.val() + b.val()); } +inline TimeStamp operator -(const TimeStamp & a, const TimeStamp & b) { return TimeStamp(a.val() - b.val()); } + +class ClockSystem +{ +public: + static int64_t now(); + static int64_t adjustTick2Sec(int64_t tick) { return tick; } +}; + +template <typename ClockT> +class StopWatchT : public ClockT +{ +public: + StopWatchT(void) : _startTime(), _stopTime() { } + + void start() { _startTime = this->now(); _stopTime = _startTime; } + void stop() { _stopTime = this->now(); } + + TimeStamp elapsedAdjusted() const { return this->adjustTick2Sec(elapsed()); } + TimeStamp startTime() const { return this->adjustTick2Sec(_startTime); } + TimeStamp stopTime() const { return this->adjustTick2Sec(_stopTime); } + + TimeStamp elapsed() const { + TimeStamp diff(_stopTime - _startTime); + return this->adjustTick2Sec((diff > 0) ? diff : TimeStamp(0)); + } +private: + TimeStamp _startTime; + TimeStamp _stopTime; +}; + +typedef StopWatchT<ClockSystem> TickStopWatch; +typedef TickStopWatch StopWatch; + +} + diff --git a/fastos/src/vespa/fastos/types.h b/fastos/src/vespa/fastos/types.h new file mode 100644 index 00000000000..5e2b0e68f1d --- /dev/null +++ b/fastos/src/vespa/fastos/types.h @@ -0,0 +1,379 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//**************************************************************************** +/** + * @file + * Type definitions used in FastOS. + * @author Oivind H. Danielsen + * @date Creation date: 2000-01-18 + *****************************************************************************/ + + +#pragma once + +#ifndef FASTOS_AUTOCONF /* Are we in the autoconf stage? */ +#include <vespa/fastos/autoconf.h> +#endif + +/** + * @def __STDC_LIMIT_MACROS + * According to C99, C++ implementations will only define UINT64_MAX + * etc when __STDC_LIMIT_MACROS is defined when including stdint.h. + * UINT64_C etc will only be defined when __STDC_CONSTANT_MACROS is + * defined. Since this file can be included from any of the files + * below, we define the behaviour here. + */ +#ifndef __STDC_LIMIT_MACROS +#define __STDC_LIMIT_MACROS 1 +#endif +#ifndef __STDC_CONSTANT_MACROS +#define __STDC_CONSTANT_MACROS 1 +#endif + +#include <assert.h> + +#include <pthread.h> +#include <sys/mman.h> + +#ifdef __TYPES_H_PTHREAD_U98 +#undef __USE_UNIX98 +#endif + +#include <sys/types.h> +#include <sys/uio.h> +#include <sys/param.h> +#include <sys/wait.h> +#include <sys/utsname.h> +#include <rpc/types.h> +#include <stdarg.h> +#include <ctype.h> + +#ifndef __USE_UNIX98 +#define __TYPES_H_UNISTD_U98 +#define __USE_UNIX98 +#endif +#include <unistd.h> +#ifdef __TYPES_H_UNISTD_U98 +#undef __USE_UNIX98 +#endif + +#include <dirent.h> + +#include <sys/socket.h> + +#ifndef SHUT_RDWR +#define SHUT_RDWR 2 +#endif + +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> + +#include <sys/resource.h> +#include <signal.h> + +#include <sys/time.h> +#include <time.h> + +#define FASTOS_EMFILE_VERIFIED (-1) +#ifdef EMFILE +#undef FASTOS_EMFILE_VERIFIED +#define FASTOS_EMFILE_VERIFIED EMFILE +#endif + +#define FASTOS_ENFILE_VERIFIED (-1) +#ifdef ENFILE +#undef FASTOS_ENFILE_VERIFIED +#define FASTOS_ENFILE_VERIFIED ENFILE +#endif + +#ifndef __USE_GNU +#define __USE_GNU /* for O_DIRECT define */ +#define __TYPES_H_DIRECTIO_GNU +#endif + +#include <fcntl.h> + +#ifdef __TYPES_H_DIRECTIO_GNU +#undef __USE_GNU /* for O_DIRECT define */ +#endif + +typedef caddr_t FastOS_SockOptValP; + +#ifndef FASTOS_AUTOCONF +#if defined(FASTOS_HAVE_SOCKLEN_T) +typedef socklen_t FastOS_SocketLen; +#else +typedef int FastOS_SocketLen; +#endif +#endif /* FASTOS_AUTOCONF */ + + +#include <stdio.h> +#include <stdlib.h> + +#include <string.h> +#include <errno.h> +#include <sys/stat.h> +#include <limits.h> +#include <float.h> + +#include <netinet/tcp.h> + +#define FASTOS_PREFIX(a) FastOS_##a + +#ifndef NULL +#define NULL 0 +#endif + +#ifndef FASTOS_AUTOCONF + +#include <inttypes.h> + +/** + * On UNIX we use the [long long] type for 64bit integers. + */ +#ifndef FASTOS_HAVE_INT64_T +typedef long long int64_t; +#endif + +#ifndef FASTOS_HAVE_UINT64_T +typedef unsigned long long uint64_t; +#endif + +#ifndef FASTOS_HAVE_INT32_T +typedef int int32_t; +#endif + +#ifndef FASTOS_HAVE_UINT32_T +typedef unsigned int uint32_t; +#endif + +#ifndef FASTOS_HAVE_INT16_T +typedef short int int16_t; +#endif + +#ifndef FASTOS_HAVE_UINT16_T +typedef unsigned short int uint16_t; +#endif + +#ifndef FASTOS_HAVE_INT8_T +typedef signed char int8_t; +#endif + +#ifndef FASTOS_HAVE_UINT8_T +typedef unsigned char uint8_t; +#endif + +#ifndef INT64_C +#ifdef FASTOS_64BIT_LONG +#define INT64_C(c) c ## L +#else +# warning "You need to define INT64_C or find a header that defines this macro" +#endif +#endif /* INT64_C */ + +#ifndef UINT64_C +#ifdef FASTOS_64BIT_LONG +#define UINT64_C(c) c ## UL +#else +#define UINT64_C(c) c ## ULL +#endif +#endif /* UINT64_C */ + + +#ifndef INT8_MIN +#define INT8_MIN (-128) +#endif + +#ifndef INT16_MIN +#define INT16_MIN (-32767-1) +#endif + +#ifndef INT32_MIN +#define INT32_MIN (-2147483647-1) +#endif + +#ifndef INT64_MIN +#define INT64_MIN (-INT64_C(9223372036854775807)-1) +#endif + +#ifndef INT8_MAX +#define INT8_MAX (127) +#endif + +#ifndef INT16_MAX +#define INT16_MAX (32767) +#endif + +#ifndef INT32_MAX +#define INT32_MAX (2147483647) +#endif + +#ifndef INT64_MAX +#define INT64_MAX (INT64_C(9223372036854775807)) +#endif + +#ifndef UINT8_MAX +#define UINT8_MAX (255U) +#endif + +#ifndef UINT16_MAX +#define UINT16_MAX (65535U) +#endif + +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + +#ifndef UINT64_MAX +#define UINT64_MAX (UINT64_C(18446744073709551615)) +#endif + +#include <getopt.h> + +#endif /* FASTOS_AUTOCONF */ + +#ifndef SHUT_WR +#define SHUT_WR 1 +#endif + +/* 64bit printf specifiers */ +#ifdef FASTOS_64BIT_LONG +#ifndef PRId64 +#define PRId64 "ld" +#endif + +#ifndef PRIu64 +#define PRIu64 "lu" +#endif + +#ifndef PRIo64 +#define PRIo64 "lo" +#endif + +#ifndef PRIx64 +#define PRIx64 "lx" +#endif + +#ifndef PRIX64 +#define PRIX64 "lX" +#endif + +#ifndef SCNd64 +#define SCNd64 "ld" +#endif + +#ifndef SCNu64 +#define SCNu64 "lu" +#endif + +#ifndef SCNo64 +#define SCNo64 "lo" +#endif + +#ifndef SCNx64 +#define SCNx64 "lx" +#endif + +#ifndef SCNX64 +#define SCNX64 "lX" +#endif + +#else /* ! FASTOS_64BIT_LONG */ + +#ifndef PRId64 +#define PRId64 "lld" +#endif + +#ifndef PRIu64 +#define PRIu64 "llu" +#endif + +#ifndef PRIo64 +#define PRIo64 "llo" +#endif + +#ifndef PRIx64 +#define PRIx64 "llx" +#endif + +#ifndef PRIX64 +#define PRIX64 "llX" +#endif + +#ifndef SCNd64 +#define SCNd64 "lld" +#endif + +#ifndef SCNu64 +#define SCNu64 "llu" +#endif + +#ifndef SCNo64 +#define SCNo64 "llo" +#endif + +#ifndef SCNx64 +#define SCNx64 "llx" +#endif + +#ifndef SCNX64 +#define SCNX64 "llX" +#endif + +#endif /* FASTOS_64BIT_LONG */ + +#ifndef PRId32 +#define PRId32 "d" +#endif + +#ifndef PRIu32 +#define PRIu32 "u" +#endif + +#ifndef PRIo32 +#define PRIo32 "o" +#endif + +#ifndef PRIx32 +#define PRIx32 "x" +#endif + +#ifndef PRIX32 +#define PRIX32 "X" +#endif + +#ifndef SCNd32 +#define SCNd32 "d" +#endif + +#ifndef SCNu32 +#define SCNu32 "u" +#endif + +#ifndef SCNo32 +#define SCNo32 "o" +#endif + +#ifndef SCNx32 +#define SCNx32 "x" +#endif + +#ifndef SCNX32 +#define SCNX32 "X" +#endif + + +typedef pthread_t FastOS_ThreadId; + +#define FASTOS_EXTERNC extern "C" +#define FASTOS_KLASS class + +#define FASTOS_IPCMSGBUF_MAXSIZE (200) + +#if defined(FASTOS_ICONV_NOT_CONST) +#define LIBICONV_CONST +#else +#define LIBICONV_CONST const +#endif + diff --git a/fastos/src/vespa/fastos/unix_app.cpp b/fastos/src/vespa/fastos/unix_app.cpp new file mode 100644 index 00000000000..3d09a2b5455 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_app.cpp @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-01-18 +* @file +* Implementation of FastOS_UNIX_Application methods. +*****************************************************************************/ + +#include <vespa/fastos/app.h> +#include <vespa/fastos/time.h> +#include <vespa/fastos/process.h> +#include <vespa/fastos/unix_ipc.h> + + +FastOS_UNIX_Application::FastOS_UNIX_Application () + : _processStarter(NULL), + _ipcHelper(NULL) +{ +} + +FastOS_UNIX_Application::~FastOS_UNIX_Application() +{ +} + +extern "C" +{ +extern char **environ; +}; + +unsigned int FastOS_UNIX_Application::GetCurrentProcessId () +{ + return static_cast<unsigned int>(getpid()); +} + + +bool FastOS_UNIX_Application:: +SendIPCMessage (FastOS_UNIX_Process *xproc, const void *buffer, + int length) +{ + if(_ipcHelper == NULL) + return false; + return _ipcHelper->SendMessage(xproc, buffer, length); +} + + +bool FastOS_UNIX_Application:: +SendParentIPCMessage (const void *data, size_t length) +{ + if(_ipcHelper == NULL) + return false; + return _ipcHelper->SendMessage(NULL, data, length); +} + + +bool FastOS_UNIX_Application::PreThreadInit () +{ + bool rc = true; + if (FastOS_ApplicationInterface::PreThreadInit()) { + // Ignore SIGPIPE + struct sigaction act; + act.sa_handler = SIG_IGN; + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + sigaction(SIGPIPE, &act, NULL); + + if (useProcessStarter()) { + _processStarter = new FastOS_UNIX_ProcessStarter(this); + if (!_processStarter->Start()) { + rc = false; + fprintf(stderr, "could not start FastOS_UNIX_ProcessStarter\n"); + } + } + } else { + rc = false; + fprintf(stderr, "FastOS_ApplicationInterface::PreThreadInit failed\n"); + } + return rc; +} + +bool FastOS_UNIX_Application::Init () +{ + bool rc = false; + + if(FastOS_ApplicationInterface::Init()) + { + int ipcDescriptor = -1; + + char *env = getenv("FASTOS_IPC_PARENT"); + if(env != NULL) + { + int commaCount=0; + int notDigitCount=0; + char *p = env; + while(*p != '\0') + { + if(*p == ',') + commaCount++; + else if((*p < '0') || (*p > '9')) + notDigitCount++; + p++; + } + + if((commaCount == 2) && (notDigitCount == 0)) + { + int ppid, gppid, descriptor; + sscanf(env, "%d,%d,%d", &ppid, &gppid, &descriptor); + + if(ppid == getppid() && (descriptor != -1)) + { + ipcDescriptor = descriptor; + } + } + } + if (useIPCHelper()) { + _ipcHelper = new FastOS_UNIX_IPCHelper(this, ipcDescriptor); + GetThreadPool()->NewThread(_ipcHelper); + } + + rc = true; + } + + return rc; +} + +void FastOS_UNIX_Application::Cleanup () +{ + if(_ipcHelper != NULL) + _ipcHelper->Exit(); + + if (_processStarter != NULL) { + if (_processListMutex) ProcessLock(); + _processStarter->Stop(); + if (_processListMutex) ProcessUnlock(); + delete _processStarter; + _processStarter = NULL; + } + + FastOS_ApplicationInterface::Cleanup(); +} + +FastOS_UNIX_ProcessStarter * +FastOS_UNIX_Application::GetProcessStarter () +{ + return _processStarter; +} + +void FastOS_UNIX_Application:: +AddToIPCComm (FastOS_UNIX_Process *process) +{ + if(_ipcHelper != NULL) + _ipcHelper->AddProcess(process); +} + +void FastOS_UNIX_Application:: +RemoveFromIPCComm (FastOS_UNIX_Process *process) +{ + if(_ipcHelper != NULL) + _ipcHelper->RemoveProcess(process); +} diff --git a/fastos/src/vespa/fastos/unix_app.h b/fastos/src/vespa/fastos/unix_app.h new file mode 100644 index 00000000000..6b0caac41e4 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_app.h @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Class definitions for FastOS_UNIX_Application. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + + +#include <vespa/fastos/types.h> +#include <vespa/fastos/app.h> + +class FastOS_UNIX_ProcessStarter; +class FastOS_UNIX_IPCHelper; +class FastOS_UNIX_Process; + +/** + * This is the generic UNIX implementation of @ref FastOS_ApplicationInterface + */ +class FastOS_UNIX_Application : public FastOS_ApplicationInterface +{ +private: + FastOS_UNIX_Application(const FastOS_UNIX_Application&); + FastOS_UNIX_Application& operator=(const FastOS_UNIX_Application&); + + FastOS_UNIX_ProcessStarter *_processStarter; + FastOS_UNIX_IPCHelper *_ipcHelper; + +protected: + virtual bool PreThreadInit (); +public: + FastOS_UNIX_Application (); + virtual ~FastOS_UNIX_Application(); + + int GetOpt (const char *optionsString, + const char* &optionArgument, + int &optionIndex) + { + optind = optionIndex; + + int rc = getopt(_argc, _argv, optionsString); + optionArgument = optarg; + optionIndex = optind; + return rc; + } + + int GetOptLong(const char *optionsString, + const char* &optionArgument, + int &optionIndex, + const struct option *longopts, + int *longindex) + { + optind = optionIndex; + + int rc = getopt_long(_argc, _argv, optionsString, + longopts, + longindex); + + optionArgument = optarg; + optionIndex = optind; + return rc; + } + + static unsigned int GetCurrentProcessId (); + + FastOS_UNIX_ProcessStarter *GetProcessStarter (); + virtual bool Init (); + virtual void Cleanup (); + bool SendParentIPCMessage (const void *data, size_t length); + bool SendIPCMessage (FastOS_UNIX_Process *xproc, const void *buffer, + int length); + void AddToIPCComm (FastOS_UNIX_Process *process); + void RemoveFromIPCComm (FastOS_UNIX_Process *process); +}; + + diff --git a/fastos/src/vespa/fastos/unix_cond.cpp b/fastos/src/vespa/fastos/unix_cond.cpp new file mode 100644 index 00000000000..097087ec799 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_cond.cpp @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/cond.h> + +FastOS_UNIX_Cond::FastOS_UNIX_Cond(void) + : FastOS_CondInterface(), + _cond() +{ + pthread_cond_init(&_cond, NULL); +} + +FastOS_UNIX_Cond::~FastOS_UNIX_Cond(void) +{ + pthread_cond_destroy(&_cond); +} + +void +FastOS_UNIX_Cond::Wait(void) +{ + pthread_cond_wait(&_cond, &_mutex); +} + +bool +FastOS_UNIX_Cond::TimedWait(int milliseconds) +{ + struct timeval currentTime; + struct timespec absTime; + int error; + + gettimeofday(¤tTime, NULL); + + int64_t ns = (static_cast<int64_t>(currentTime.tv_sec) * + static_cast<int64_t>(1000 * 1000 * 1000) + + static_cast<int64_t>(currentTime.tv_usec) * + static_cast<int64_t>(1000) + + static_cast<int64_t>(milliseconds) * + static_cast<int64_t>(1000 * 1000)); + + absTime.tv_sec = static_cast<int> + (ns / static_cast<int64_t>(1000 * 1000 * 1000)); + absTime.tv_nsec = static_cast<int> + (ns % static_cast<int64_t>(1000 * 1000 * 1000)); + + error = pthread_cond_timedwait(&_cond, &_mutex, &absTime); + return error == 0; +} diff --git a/fastos/src/vespa/fastos/unix_cond.h b/fastos/src/vespa/fastos/unix_cond.h new file mode 100644 index 00000000000..1f4ef336d93 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_cond.h @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Class definition and implementation for FastOS_UNIX_Cond. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + +#include <vespa/fastos/cond.h> + + +class FastOS_UNIX_Cond : public FastOS_CondInterface +{ +private: + FastOS_UNIX_Cond(const FastOS_UNIX_Cond &); + FastOS_UNIX_Cond& operator=(const FastOS_UNIX_Cond &); + + pthread_cond_t _cond; + +public: + FastOS_UNIX_Cond (void); + + ~FastOS_UNIX_Cond(void); + + void Wait(void); + + bool TimedWait(int milliseconds); + + void Signal(void) + { + pthread_cond_signal(&_cond); + } + + void Broadcast(void) + { + pthread_cond_broadcast(&_cond); + } +}; + + diff --git a/fastos/src/vespa/fastos/unix_dynamiclibrary.cpp b/fastos/src/vespa/fastos/unix_dynamiclibrary.cpp new file mode 100644 index 00000000000..bd623495e1a --- /dev/null +++ b/fastos/src/vespa/fastos/unix_dynamiclibrary.cpp @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/dynamiclibrary.h> +#include <vespa/fastos/file.h> + +#include <dlfcn.h> + +namespace { +const std::string FASTOS_DYNLIB_PREFIX("lib"); +const std::string FASTOS_DYNLIB_SUFFIX(".so"); +const std::string FASTOS_DYNLIB_SUFPREFIX(".so."); + +bool hasValidSuffix(const std::string & s) +{ + return (s.rfind(FASTOS_DYNLIB_SUFFIX) == (s.size() - FASTOS_DYNLIB_SUFFIX.size())) + || (s.rfind(FASTOS_DYNLIB_SUFPREFIX) != std::string::npos); +} + +} + +void +FastOS_UNIX_DynamicLibrary::SetLibName(const char *libname) +{ + if (libname != NULL) { + _libname = libname; + if ( ! hasValidSuffix(_libname)) { + _libname.append(FASTOS_DYNLIB_SUFFIX); + } + } else { + _libname = ""; + } +} + +bool +FastOS_UNIX_DynamicLibrary::NormalizeLibName(void) +{ + bool returnCode = false; + std::string::size_type pathPos = _libname.rfind(FastOS_File::GetPathSeparator()[0]); + std::string tmp = (pathPos != std::string::npos) + ? _libname.substr(pathPos+1) + : _libname; + if (tmp.find(FASTOS_DYNLIB_PREFIX) != 0) { + tmp = FASTOS_DYNLIB_PREFIX + tmp; + if (pathPos != std::string::npos) { + tmp = _libname.substr(0, pathPos); + } + SetLibName(tmp.c_str()); + returnCode = true; + } + + return returnCode; +} + +bool +FastOS_UNIX_DynamicLibrary::Close() +{ + bool retcode = true; + + if (IsOpen()) { + retcode = (dlclose(_handle) == TRUE); + if (retcode) + _handle = NULL; + } + + return retcode; +} + +FastOS_UNIX_DynamicLibrary::FastOS_UNIX_DynamicLibrary(const char *libname) : + _handle(NULL), + _libname("") +{ + SetLibName(libname); +} + +FastOS_UNIX_DynamicLibrary::~FastOS_UNIX_DynamicLibrary() +{ + Close(); +} + +bool +FastOS_UNIX_DynamicLibrary::Open(const char *libname) +{ + if (! Close()) + return false; + if (libname != NULL) { + SetLibName(libname); + } + + _handle = dlopen(_libname.c_str(), RTLD_NOW); + + if (_handle == NULL) { + // Prepend "lib" if neccessary... + if (NormalizeLibName()) { + // ...try to open again if a change was made. + _handle = dlopen(_libname.c_str(), RTLD_NOW); + } + } + + return (_handle != NULL); +} + +void * +FastOS_UNIX_DynamicLibrary::GetSymbol(const char *symbol) const +{ + return dlsym(_handle, symbol); +} + +std::string +FastOS_UNIX_DynamicLibrary::GetLastErrorString() const +{ + const char *errorString = dlerror(); + std::string e; + if (errorString != NULL) { + e = errorString; + } + + return e; +} diff --git a/fastos/src/vespa/fastos/unix_dynamiclibrary.h b/fastos/src/vespa/fastos/unix_dynamiclibrary.h new file mode 100644 index 00000000000..7972a6c98fb --- /dev/null +++ b/fastos/src/vespa/fastos/unix_dynamiclibrary.h @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +*************************************************************-*C++-*- +* @author Eyvind Bernhardsen +* @date Creation date: 2003-07-02 +* @file +* Class definitions for FastOS_Unix_DynamicLibrary +*********************************************************************/ + + + +#pragma once + + +#include <vespa/fastos/types.h> + +class FastOS_UNIX_DynamicLibrary : public FastOS_DynamicLibraryInterface +{ +private: + FastOS_UNIX_DynamicLibrary(const FastOS_UNIX_DynamicLibrary&); + FastOS_UNIX_DynamicLibrary& operator=(const FastOS_UNIX_DynamicLibrary&); + + void *_handle; + std::string _libname; + +public: + FastOS_UNIX_DynamicLibrary(const char *libname = NULL); + ~FastOS_UNIX_DynamicLibrary(); + + void SetLibName(const char *libname); + bool NormalizeLibName(void); + bool Close(); + bool Open(const char *libname = NULL); + void * GetSymbol(const char *symbol) const; + std::string GetLastErrorString() const; + const char * GetLibName() const { return _libname.c_str(); } + bool IsOpen() const { return (_handle != NULL); } +}; + + diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/fastos/src/vespa/fastos/unix_file.cpp new file mode 100644 index 00000000000..bf8f337fdab --- /dev/null +++ b/fastos/src/vespa/fastos/unix_file.cpp @@ -0,0 +1,527 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-01-18 +* @file +* Implementation of FastOS_UNIX_File methods. +*****************************************************************************/ + +#include <vespa/fastos/file.h> +#include <sys/vfs.h> +#include <sstream> +#include <stdexcept> + +bool +FastOS_UNIX_File::SetPosition(int64_t desiredPosition) +{ + int64_t position = lseek(_filedes, desiredPosition, SEEK_SET); + + return (position == desiredPosition); +} + + +int64_t +FastOS_UNIX_File::GetPosition(void) +{ + return lseek(_filedes, 0, SEEK_CUR); +} + + +bool +FastOS_UNIX_File::Stat(const char *filename, + FastOS_StatInfo *statInfo) +{ + bool rc = false; + + struct stat stbuf; + int lstatres; + + do { + lstatres = lstat(filename, &stbuf); + } while (lstatres == -1 && errno == EINTR); + if (lstatres == 0) { + statInfo->_error = FastOS_StatInfo::Ok; + statInfo->_isRegular = S_ISREG(stbuf.st_mode); + statInfo->_isDirectory = S_ISDIR(stbuf.st_mode); + statInfo->_size = static_cast<int64_t>(stbuf.st_size); + statInfo->_modifiedTime = stbuf.st_mtime; + statInfo->_modifiedTimeNS = stbuf.st_mtim.tv_sec; + statInfo->_modifiedTimeNS *= 1000000000; + statInfo->_modifiedTimeNS += stbuf.st_mtim.tv_nsec; + rc = true; + } else { + if (errno == ENOENT) + statInfo->_error = FastOS_StatInfo::FileNotFound; + else + statInfo->_error = FastOS_StatInfo::Unknown; + } + + return rc; +} + + +int FastOS_UNIX_File::GetMaximumFilenameLength (const char *pathName) +{ + return pathconf(pathName, _PC_NAME_MAX); +} + +int FastOS_UNIX_File::GetMaximumPathLength(const char *pathName) +{ + return pathconf(pathName, _PC_PATH_MAX); +} + +bool +FastOS_UNIX_File::MakeDirectory (const char *name) +{ + return (mkdir(name, 0775) == 0); +} + + +void +FastOS_UNIX_File::RemoveDirectory (const char *name) +{ + if ((rmdir(name) != 0) && (ERR_ENOENT != GetLastError())) { + std::ostringstream os; + os << "Remove of directory '" << name << "' failed with error :'" << getLastErrorString() << "'"; + throw std::runtime_error(os.str()); + } +} + + +std::string +FastOS_UNIX_File::getCurrentDirectory(void) +{ + std::string res; + int maxPathLen = FastOS_File::GetMaximumPathLength("."); + if (maxPathLen == -1) + maxPathLen = 16384; + else if (maxPathLen < 512) + maxPathLen = 512; + + char *currentDir = new char [maxPathLen + 1]; + + if (getcwd(currentDir, maxPathLen) != NULL) + { + res = currentDir; + } + delete [] currentDir; + + return res; +} + + +unsigned int +FastOS_UNIX_File::CalcAccessFlags(unsigned int openFlags) +{ + unsigned int accessFlags=0; + + if ((openFlags & (FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_DIRECTIO)) != 0) { + if ((openFlags & FASTOS_FILE_OPEN_WRITE) != 0) { + // Open for reading and writing + accessFlags = O_RDWR; + } else { + // Open for reading only + accessFlags = O_RDONLY; + } + } else { + // Open for writing only + accessFlags = O_WRONLY; + } + + if (((openFlags & FASTOS_FILE_OPEN_EXISTING) == 0) && + ((openFlags & FASTOS_FILE_OPEN_WRITE) != 0)) { + // Create file if it does not exist + accessFlags |= O_CREAT; + } + +#if defined(O_SYNC) + if ((openFlags & FASTOS_FILE_OPEN_SYNCWRITES) != 0) + accessFlags |= O_SYNC; +#elif defined(O_FSYNC) + if ((openFlags & FASTOS_FILE_OPEN_SYNCWRITES) != 0) + accessFlags |= O_FSYNC; +#endif + + if ((openFlags & FASTOS_FILE_OPEN_DIRECTIO) != 0) { + accessFlags |= O_DIRECT | O_DSYNC | O_RSYNC; + } + + if ((openFlags & FASTOS_FILE_OPEN_TRUNCATE) != 0) { + // Truncate file on open + accessFlags |= O_TRUNC; + } + return accessFlags; +} + +bool +FastOS_UNIX_File::Open(unsigned int openFlags, const char *filename) +{ + bool rc = false; + assert(_filedes == -1); + + if ((openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { + FILE *file; + + switch(openFlags & FASTOS_FILE_OPEN_STDFLAGS) { + case FASTOS_FILE_OPEN_STDIN: + file = stdin; + SetFileName("stdin"); + break; + + case FASTOS_FILE_OPEN_STDOUT: + file = stdout; + SetFileName("stdout"); + break; + + case FASTOS_FILE_OPEN_STDERR: + file = stderr; + SetFileName("stderr"); + break; + + default: + file = NULL; + fprintf(stderr, "Invalid open-flags %08X\n", openFlags); + abort(); + } + + _filedes = file->_fileno; + _openFlags = openFlags; + rc = true; + } else { + if (filename != NULL) + SetFileName(filename); + + unsigned int accessFlags = CalcAccessFlags(openFlags); + + _filedes = open(_filename, accessFlags, 0664); + + rc = (_filedes != -1); + + if (rc) { + _openFlags = openFlags; + if (_mmapEnabled) { + int64_t filesize = GetSize(); + size_t mlen = static_cast<size_t>(filesize); + if (static_cast<int64_t>(mlen) == filesize && mlen > 0) { + void *mbase = mmap(NULL, mlen, PROT_READ, MAP_SHARED | _mmapFlags, _filedes, static_cast<off_t>(0)); + if (static_cast<void *>(mbase) != reinterpret_cast<void *>(-1)) { + int fadviseOptions = getFAdviseOptions(); + int eCode(0); + if (POSIX_FADV_RANDOM == fadviseOptions) { + eCode = posix_madvise(mbase, mlen, POSIX_MADV_RANDOM); + } else if (POSIX_FADV_SEQUENTIAL == fadviseOptions) { + eCode = posix_madvise(mbase, mlen, POSIX_MADV_SEQUENTIAL); + } + if (eCode != 0) { + fprintf(stderr, "Failed: posix_madvise(%p, %ld, %d) = %d\n", mbase, mlen, fadviseOptions, eCode); + } + _mmapbase = mbase; + _mmaplen = mlen; + } else { + std::ostringstream os; + os << "mmap of file '" << GetFileName() << "' with flags '" << std::hex << (MAP_SHARED | _mmapFlags) << std::dec + << "' failed with error :'" << getErrorString(GetLastOSError()) << "'"; + throw std::runtime_error(os.str()); + } + } + } + } + + } + + return rc; +} + +void FastOS_UNIX_File::dropFromCache() const +{ + posix_fadvise(_filedes, 0, 0, POSIX_FADV_DONTNEED); +} + + +bool +FastOS_UNIX_File::Close(void) +{ + bool ok = true; + + if (_filedes >= 0) { + if ((_openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) + ok = true; + else { + do { + ok = (close(_filedes) == 0); + } while (!ok && errno == EINTR); + } + + if (_mmapbase != NULL) { + munmap(static_cast<char *>(_mmapbase), _mmaplen); + _mmapbase = NULL; + _mmaplen = 0; + } + + _filedes = -1; + } + + _openFlags = 0; + + return ok; +} + + +int64_t +FastOS_UNIX_File::GetSize(void) +{ + int64_t fileSize=-1; + struct stat stbuf; + + assert(IsOpened()); + + int res = fstat(_filedes, &stbuf); + + if (res == 0) + fileSize = stbuf.st_size; + + return fileSize; +} + + +time_t +FastOS_UNIX_File::GetModificationTime(void) +{ + struct stat stbuf; + int res; + + assert(IsOpened()); + + res = fstat(_filedes, &stbuf); + assert(res == 0); + (void) res; + + return stbuf.st_mtime; +} + + +bool +FastOS_UNIX_File::Delete(const char *name) +{ + return (unlink(name) == 0); +} + + +bool +FastOS_UNIX_File::Delete(void) +{ + assert(!IsOpened()); + assert(_filename != NULL); + + return (unlink(_filename) == 0); +} + +bool FastOS_UNIX_File::Rename (const char *currentFileName, + const char *newFileName) +{ + bool rc = false; + + // Enforce documentation. If the destination file exists, + // fail Rename. + FastOS_StatInfo statInfo; + if (!FastOS_File::Stat(newFileName, &statInfo)) + { + rc = (rename(currentFileName, newFileName) == 0); + } else { + errno = EEXIST; + } + return rc; +} + +bool +FastOS_UNIX_File::Sync(void) +{ + assert(IsOpened()); + + return (fsync(_filedes) == 0); +} + + +bool +FastOS_UNIX_File::SetSize(int64_t newSize) +{ + bool rc = false; + + if (ftruncate(_filedes, static_cast<off_t>(newSize)) == 0) + rc = SetPosition(newSize); + + return rc; +} + + +FastOS_File::Error +FastOS_UNIX_File::TranslateError (const int osError) +{ + switch(osError) { + case ENOENT: return ERR_NOENT; // No such file or directory + case ENOMEM: return ERR_NOMEM; // Not enough memory + case EACCES: return ERR_ACCES; // Permission denied + case EEXIST: return ERR_EXIST; // File exists + case EINVAL: return ERR_INVAL; // Invalid argument + case ENOSPC: return ERR_NOSPC; // No space left on device + case EINTR: return ERR_INTR; // interrupt + case EAGAIN: return ERR_AGAIN; // Resource unavailable, try again + case EBUSY: return ERR_BUSY; // Device or resource busy + case EIO: return ERR_IO; // I/O error + case EPERM: return ERR_PERM; // Not owner + case ENODEV: return ERR_NODEV; // No such device + case ENXIO: return ERR_NXIO; // Device not configured + } + + if (osError == FASTOS_ENFILE_VERIFIED) + return ERR_NFILE; + + if (osError == FASTOS_EMFILE_VERIFIED) + return ERR_MFILE; + + return ERR_UNKNOWN; +} + + +std::string +FastOS_UNIX_File::getErrorString(const int osError) +{ + char errorBuf[100]; + const char *errorString = strerror_r(osError, errorBuf, sizeof(errorBuf)); + + return std::string(errorString); +} + + +int64_t FastOS_UNIX_File::GetFreeDiskSpace (const char *path) +{ + int64_t freeSpace = -1; + + struct statfs statBuf; + int statVal = -1; + statVal = statfs(path, &statBuf); + if (statVal == 0) + freeSpace = int64_t(statBuf.f_bavail) * int64_t(statBuf.f_bsize); + + return freeSpace; +} + +FastOS_UNIX_DirectoryScan::FastOS_UNIX_DirectoryScan(const char *searchPath) + : FastOS_DirectoryScanInterface(searchPath), + _statRun(false), + _isDirectory(false), + _isRegular(false), + _statName(NULL), + _statFilenameP(NULL), + _dir(NULL), + _dp(NULL) +{ + _dir = opendir(searchPath); + + const int minimumLength = 512 + 1; + const int defaultLength = 16384; + + int maxNameLength = FastOS_File::GetMaximumFilenameLength(searchPath); + int maxPathLength = FastOS_File::GetMaximumPathLength(searchPath); + int nameLength = maxNameLength + 1 + maxPathLength; + + if ((maxNameLength == -1) || + (maxPathLength == -1) || + (nameLength < minimumLength)) + { + nameLength = defaultLength; + } + + _statName = new char [nameLength + 1]; // Include null + + strcpy(_statName, searchPath); + strcat(_statName, "/"); + + _statFilenameP = &_statName[strlen(_statName)]; +} + + +FastOS_UNIX_DirectoryScan::~FastOS_UNIX_DirectoryScan(void) +{ + if (_dir != NULL) { + closedir(_dir); + _dir = NULL; + } + delete [] _statName; +} + + +bool +FastOS_UNIX_DirectoryScan::ReadNext(void) +{ + bool rc = false; + + _statRun = false; + + if (_dir != NULL) { + _dp = readdir(_dir); + rc = _dp != NULL; + } + + return rc; +} + + +void +FastOS_UNIX_DirectoryScan::DoStat(void) +{ + struct stat stbuf; + + assert(_dp != NULL); + + strcpy(_statFilenameP, _dp->d_name); + + if (lstat(_statName, &stbuf) == 0) { + _isRegular = S_ISREG(stbuf.st_mode); + _isDirectory = S_ISDIR(stbuf.st_mode); + } else { + printf("lstat failed for [%s]\n", _dp->d_name); + _isRegular = false; + _isDirectory = false; + } + + _statRun = true; +} + + +bool +FastOS_UNIX_DirectoryScan::IsDirectory(void) +{ + if (!_statRun) + DoStat(); + + return _isDirectory; +} + + +bool +FastOS_UNIX_DirectoryScan::IsRegular(void) +{ + if (!_statRun) + DoStat(); + + return _isRegular; +} + + +const char * +FastOS_UNIX_DirectoryScan::GetName(void) +{ + assert(_dp != NULL); + + return static_cast<const char *>(_dp->d_name); +} + + +bool +FastOS_UNIX_DirectoryScan::IsValidScan(void) const +{ + return _dir != NULL; +} diff --git a/fastos/src/vespa/fastos/unix_file.h b/fastos/src/vespa/fastos/unix_file.h new file mode 100644 index 00000000000..7fc471374b3 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_file.h @@ -0,0 +1,161 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-01-18 +* @file +* Class definitions for FastOS_UNIX_File and FastOS_UNIX_DirectoryScan. +*****************************************************************************/ + + + +#pragma once + + +#include <vespa/fastos/file.h> + +/** + * This is the generic UNIX implementation of @ref FastOS_FileInterface. + */ +class FastOS_UNIX_File : public FastOS_FileInterface +{ +public: + using FastOS_FileInterface::ReadBuf; +private: + FastOS_UNIX_File(const FastOS_UNIX_File&); + FastOS_UNIX_File& operator=(const FastOS_UNIX_File&); + +protected: + void *_mmapbase; + size_t _mmaplen; + int _filedes; + int _mmapFlags; + bool _mmapEnabled; + + static unsigned int + CalcAccessFlags(unsigned int openFlags); + +public: + static bool Rename (const char *currentFileName, const char *newFileName); + bool Rename (const char *newFileName) + { return FastOS_FileInterface::Rename(newFileName); } + + static bool Stat(const char *filename, FastOS_StatInfo *statInfo); + static bool MakeDirectory(const char *name); + static void RemoveDirectory(const char *name); + + static std::string + getCurrentDirectory(void); + + static bool SetCurrentDirectory (const char *pathName) { + return (chdir(pathName) == 0); + } + static int GetMaximumFilenameLength (const char *pathName); + static int GetMaximumPathLength (const char *pathName); + + FastOS_UNIX_File(const char *filename=NULL) + : FastOS_FileInterface(filename), + _mmapbase(NULL), + _mmaplen(0), + _filedes(-1), + _mmapFlags(0), + _mmapEnabled(false) + { + } + + char *ToString(void); + + virtual bool Open(unsigned int openFlags, const char *filename=NULL); + + virtual bool Close(void); + + virtual bool IsOpened(void) const + { + return _filedes >= 0; + } + + virtual void enableMemoryMap(int flags) { + _mmapEnabled = true; + _mmapFlags = flags; + } + + virtual void *MemoryMapPtr(int64_t position) const + { + if (_mmapbase != NULL) { + if (position < int64_t(_mmaplen)) { + return static_cast<void *>(static_cast<char *>(_mmapbase) + position); + } else { // This is an indication that the file size has changed and a remap/reopen must be done. + return NULL; + } + } else { + return NULL; + } + } + + virtual bool IsMemoryMapped(void) const { + return _mmapbase != NULL; + } + + virtual bool SetPosition(int64_t desiredPosition); + + virtual int64_t GetPosition(void); + + virtual int64_t GetSize(void); + virtual time_t GetModificationTime(void); + + static bool Delete(const char *filename); + virtual bool Delete(void); + + virtual bool Sync(void); + + virtual bool SetSize(int64_t newSize); + + static int GetLastOSError(void) + { + return errno; + }; + + static Error TranslateError(const int osError); + + static std::string + getErrorString(const int osError); + + static int64_t GetFreeDiskSpace (const char *path); + void dropFromCache() const override; +}; + + + +/** + * This is the generic UNIX implementation of @ref FastOS_DirectoryScan. + */ +class FastOS_UNIX_DirectoryScan : public FastOS_DirectoryScanInterface +{ +private: + FastOS_UNIX_DirectoryScan(const FastOS_UNIX_DirectoryScan&); + FastOS_UNIX_DirectoryScan& operator=(const FastOS_UNIX_DirectoryScan&); + + bool _statRun; + bool _isDirectory; + bool _isRegular; + char *_statName; + char *_statFilenameP; + + void DoStat(void); + +protected: + DIR *_dir; + struct dirent *_dp; + +public: + FastOS_UNIX_DirectoryScan(const char *searchPath); + ~FastOS_UNIX_DirectoryScan(void); + + bool ReadNext(void); + bool IsDirectory(void); + bool IsRegular(void); + + const char *GetName(void); + + bool IsValidScan(void) const; +}; diff --git a/fastos/src/vespa/fastos/unix_ipc.cpp b/fastos/src/vespa/fastos/unix_ipc.cpp new file mode 100644 index 00000000000..98833661a38 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_ipc.cpp @@ -0,0 +1,697 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/unix_ipc.h> + +#include <vespa/fastos/ringbuffer.h> + +FastOS_UNIX_IPCHelper:: +FastOS_UNIX_IPCHelper (FastOS_ApplicationInterface *app, + int descriptor) + : _lock(), + _exitFlag(false), + _app(app), + _appParentIPCDescriptor() +{ + _appParentIPCDescriptor._fd = descriptor; + _wakeupPipe[0] = -1; + _wakeupPipe[1] = -1; + + if(pipe(_wakeupPipe) != 0) { + perror("pipe wakeuppipe"); + exit(1); + } else { + SetBlocking(_wakeupPipe[0], false); + SetBlocking(_wakeupPipe[1], true); + } + + if(_appParentIPCDescriptor._fd != -1) { + _appParentIPCDescriptor._readBuffer.reset(new FastOS_RingBuffer(16384)); + _appParentIPCDescriptor._writeBuffer.reset(new FastOS_RingBuffer(16384)); + + SetBlocking(_appParentIPCDescriptor._fd, false); + } +} + +FastOS_UNIX_IPCHelper::~FastOS_UNIX_IPCHelper () +{ + if(_wakeupPipe[0] != -1) { + close(_wakeupPipe[0]); + } + if(_wakeupPipe[1] != -1) { + close(_wakeupPipe[1]); + } + if(_appParentIPCDescriptor._fd != -1) { + close(_appParentIPCDescriptor._fd); + } + _appParentIPCDescriptor._readBuffer.reset(); + _appParentIPCDescriptor._writeBuffer.reset(); +} + + +bool FastOS_UNIX_IPCHelper:: +DoWrite(FastOS_UNIX_Process::DescriptorHandle &desc) +{ + bool rc = true; + FastOS_RingBuffer *buffer = desc._writeBuffer.get(); + + buffer->Lock(); + int writeBytes = buffer->GetReadSpace(); + if(writeBytes > 0) + { + int bytesWritten; + do + { + bytesWritten = write(desc._fd, + buffer->GetReadPtr(), + writeBytes); + } while(bytesWritten < 0 && errno == EINTR); + + if(bytesWritten > 0) + buffer->Consume(bytesWritten); + else if(bytesWritten < 0) + { + desc.CloseHandle(); + perror("FastOS_UNIX_IPCHelper::DoWrite"); + rc = false; + } + else if(bytesWritten == 0) + desc.CloseHandle(); + } + buffer->Unlock(); + + return rc; +} + +bool FastOS_UNIX_IPCHelper:: +DoRead (FastOS_UNIX_Process::DescriptorHandle &desc) +{ + bool rc = true; + + FastOS_RingBuffer *buffer = desc._readBuffer.get(); + + buffer->Lock(); + int readBytes = buffer->GetWriteSpace(); + if(readBytes > 0) { + int bytesRead; + do { + bytesRead = read(desc._fd, buffer->GetWritePtr(), readBytes); + } while(bytesRead < 0 && errno == EINTR); + + if (bytesRead > 0) { + buffer->Produce(bytesRead); + } else if(bytesRead < 0) { + desc.CloseHandle(); + perror("FastOS_UNIX_IPCHelper::DoRead"); + rc = false; + } else if(bytesRead == 0) { + desc.CloseHandle(); + } + } + buffer->Unlock(); + + return rc; +} + +bool FastOS_UNIX_IPCHelper:: +SetBlocking (int fileDescriptor, bool doBlock) +{ + bool rc=false; + + int flags = fcntl(fileDescriptor, F_GETFL, NULL); + if (flags != -1) + { + if(doBlock) + flags &= ~O_NONBLOCK; + else + flags |= O_NONBLOCK; + rc = (fcntl(fileDescriptor, F_SETFL, flags) != -1); + } + return rc; +} + +void FastOS_UNIX_IPCHelper:: +BuildPollCheck(bool isRead, int filedes, + FastOS_RingBuffer *buffer, bool *check) +{ + if(buffer == NULL || + filedes < 0 || + buffer->GetCloseFlag()) { + *check = false; + return; + } + + bool setIt = false; + if(isRead) + setIt = (buffer->GetWriteSpace() > 0); + else + setIt = (buffer->GetReadSpace() > 0); + *check = setIt; +} + + +void FastOS_UNIX_IPCHelper:: +PerformAsyncIO(void) +{ + FastOS_ProcessInterface *node; + for(node = _app->GetProcessList(); node != NULL; node = node->_next) + { + FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); + + for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) + { + FastOS_UNIX_Process::DescriptorType type_ = + FastOS_UNIX_Process::DescriptorType(type); + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(type_); + if (desc._canRead) + (void) DoRead(desc); + if (desc._canWrite) + (void) DoWrite(desc); + } + } +} + +void FastOS_UNIX_IPCHelper:: +PerformAsyncIPCIO(void) +{ + FastOS_UNIX_Process::DescriptorHandle &desc = _appParentIPCDescriptor; + if (desc._canRead) + (void) DoRead(desc); + if (desc._canWrite) + (void) DoWrite(desc); +} + + +void FastOS_UNIX_IPCHelper:: +BuildPollChecks(void) +{ + FastOS_ProcessInterface *node; + for(node = _app->GetProcessList(); node != NULL; node = node->_next) + { + FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); + + for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) + { + FastOS_UNIX_Process::DescriptorType type_ = + FastOS_UNIX_Process::DescriptorType(type); + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(type_); + BuildPollCheck(false, desc._fd, desc._writeBuffer.get(), &desc._wantWrite); + BuildPollCheck(true, desc._fd, desc._readBuffer.get(), &desc._wantRead); + } + } + + if(_appParentIPCDescriptor._writeBuffer.get() != NULL) + BuildPollCheck(false, _appParentIPCDescriptor._fd, + _appParentIPCDescriptor._writeBuffer.get(), + &_appParentIPCDescriptor._wantWrite); + if(_appParentIPCDescriptor._readBuffer.get() != NULL) + BuildPollCheck(true, _appParentIPCDescriptor._fd, + _appParentIPCDescriptor._readBuffer.get(), + &_appParentIPCDescriptor._wantRead); +} + + +static pollfd * +__attribute__((__noinline__)) +ResizePollArray(pollfd **fds, unsigned int *allocnfds) +{ + pollfd *newfds; + unsigned int newallocnfds; + + if (*allocnfds == 0) + newallocnfds = 16; + else + newallocnfds = *allocnfds * 2; + newfds = static_cast<pollfd *>(malloc(newallocnfds * sizeof(pollfd))); + assert(newfds != NULL); + + if (*allocnfds > 0) + memcpy(newfds, *fds, sizeof(pollfd) * *allocnfds); + + if (*fds != NULL) + free(*fds); + + *fds = newfds; + newfds += *allocnfds; + *allocnfds = newallocnfds; + return newfds; +} + +void +FastOS_UNIX_IPCHelper:: +BuildPollArray(pollfd **fds, unsigned int *nfds, unsigned int *allocnfds) +{ + FastOS_ProcessInterface *node; + pollfd *rfds; + const pollfd *rfdsEnd; + int pollIdx; + + rfds = *fds; + rfdsEnd = *fds + *allocnfds; + + if (rfds >= rfdsEnd) { + rfds = ResizePollArray(fds, + allocnfds); + rfdsEnd = *fds + *allocnfds; + } + rfds->fd = _wakeupPipe[0]; + rfds->events = POLLIN; + rfds->revents = 0; + rfds++; + pollIdx = 1; + for(node = _app->GetProcessList(); node != NULL; node = node->_next) + { + FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); + + for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) + { + FastOS_UNIX_Process::DescriptorType type_ = + FastOS_UNIX_Process::DescriptorType(type); + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(type_); + + if (desc._fd >= 0 && + (desc._wantRead || desc._wantWrite)) { + if (rfds >= rfdsEnd) { + rfds = ResizePollArray(fds, + allocnfds); + rfdsEnd = *fds + *allocnfds; + } + rfds->fd = desc._fd; + rfds->events = 0; + if (desc._wantRead) + rfds->events |= POLLRDNORM; + if (desc._wantWrite) + rfds->events |= POLLWRNORM; + rfds->revents = 0; + desc._pollIdx = pollIdx; + rfds++; + pollIdx++; + } else { + desc._pollIdx = -1; + desc._canRead = false; + desc._canWrite = false; + } + } + } + + FastOS_UNIX_Process::DescriptorHandle &desc2 = _appParentIPCDescriptor; + + if (desc2._fd >= 0 && + (desc2._wantRead || desc2._wantWrite)) { + if (rfds >= rfdsEnd) { + rfds = ResizePollArray(fds, + allocnfds); + rfdsEnd = *fds + *allocnfds; + } + rfds->fd = desc2._fd; + rfds->events = 0; + if (desc2._wantRead) + rfds->events |= POLLRDNORM; + if (desc2._wantWrite) + rfds->events |= POLLWRNORM; + rfds->revents = 0; + desc2._pollIdx = pollIdx; + rfds++; + pollIdx++; + } else { + desc2._pollIdx = -1; + desc2._canRead = false; + desc2._canWrite = false; + } + *nfds = rfds - *fds; +} + + +bool +FastOS_UNIX_IPCHelper:: +SavePollArray(pollfd *fds, unsigned int nfds) +{ + FastOS_ProcessInterface *node; + + for(node = _app->GetProcessList(); node != NULL; node = node->_next) + { + FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); + + for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) + { + FastOS_UNIX_Process::DescriptorType type_ = + FastOS_UNIX_Process::DescriptorType(type); + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(type_); + + if (desc._fd >= 0 && + static_cast<unsigned int>(desc._pollIdx) < nfds) { + int revents = fds[desc._pollIdx].revents; + + if (desc._wantRead && + (revents & + (POLLIN | POLLRDNORM | POLLERR | POLLHUP | POLLNVAL)) != 0) + desc._canRead = true; + else + desc._canRead = false; + if (desc._wantWrite && + (revents & + (POLLOUT | POLLWRNORM | POLLWRBAND | POLLERR | POLLHUP | + POLLNVAL)) != 0) + desc._canWrite = true; + else + desc._canWrite = false; + } + } + } + + FastOS_UNIX_Process::DescriptorHandle &desc2 = _appParentIPCDescriptor; + + if (desc2._fd >= 0 && + static_cast<unsigned int>(desc2._pollIdx) < nfds) { + int revents = fds[desc2._pollIdx].revents; + + if ((revents & + (POLLIN | POLLRDNORM | POLLERR | POLLHUP | POLLNVAL)) != 0) + desc2._canRead = true; + else + desc2._canRead = false; + if ((revents & + (POLLOUT | POLLWRNORM | POLLWRBAND | POLLERR | POLLHUP | + POLLNVAL)) != 0) + desc2._canWrite = true; + else + desc2._canWrite = false; + } + + if ((fds[0].revents & (POLLIN | POLLERR | POLLHUP)) != 0) + return true; + else + return false; +} + + +void FastOS_UNIX_IPCHelper:: +RemoveClosingProcesses(void) +{ + // We assume that not updating maxFD isn't harmless. + + FastOS_ProcessInterface *node, *next; + + for(node = _app->GetProcessList(); node != NULL; node = next) + { + int type; + + next = node->_next; + FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); + + bool stillBusy = false; + if(!xproc->GetKillFlag()) + for(type=0; type < FastOS_UNIX_Process::TYPE_READCOUNT; type++) + { + FastOS_UNIX_Process::DescriptorType type_; + + type_ = static_cast<FastOS_UNIX_Process::DescriptorType>(type); + + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(type_); + + if (desc._fd != -1) + { + if((type_ == FastOS_UNIX_Process::TYPE_STDOUT) || + (type_ == FastOS_UNIX_Process::TYPE_STDERR) || + desc._wantWrite) + { + // We still want to use this socket. + // Make sure we don't close the socket yet. + stillBusy = true; + break; + } + } + } + + if(!stillBusy) + { + if(xproc->_closing != NULL) + { + // We already have the process lock at this point, + // so modifying the list is safe. + _app->RemoveChildProcess(node); + + for(type=0; type < FastOS_UNIX_Process::TYPE_READCOUNT; type++) + { + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(FastOS_UNIX_Process::DescriptorType(type)); + if(desc._fd != -1) + { + // No more select on this one. + // We already know wantWrite is not set + if (desc._wantRead) + desc._wantRead = false; + } + } + + // The process destructor can now proceed + xproc->_closing->ClearBusy(); + } + } + } +} + + +void FastOS_UNIX_IPCHelper:: +Run(FastOS_ThreadInterface *thisThread, void *arg) +{ + (void)arg; + (void)thisThread; + + FastOS_ProcessInterface *node; + pollfd *fds; + unsigned int nfds; + unsigned int allocnfds; + + fds = NULL; + nfds = 0; + allocnfds = 0; + for(;;) + { + // Deliver messages to from child processes and parent. + _app->ProcessLock(); + for(node = _app->GetProcessList(); node != NULL; node = node->_next) + { + FastOS_UNIX_Process *xproc = static_cast<FastOS_UNIX_Process *>(node); + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(FastOS_UNIX_Process::TYPE_IPC); + DeliverMessages(desc._readBuffer.get()); + PipeData(xproc, FastOS_UNIX_Process::TYPE_STDOUT); + PipeData(xproc, FastOS_UNIX_Process::TYPE_STDERR); + } + DeliverMessages(_appParentIPCDescriptor._readBuffer.get()); + + // Setup file descriptor sets for the next select() call + BuildPollChecks(); + + // Close and signal closing processes + RemoveClosingProcesses(); + + BuildPollArray(&fds, &nfds, &allocnfds); + + _app->ProcessUnlock(); + + _lock.Lock(); + bool exitFlag(_exitFlag); + _lock.Unlock(); + if (exitFlag) + { + if (_appParentIPCDescriptor._fd != -1) + { + if(_appParentIPCDescriptor._wantWrite) + { + // printf("still data to write\n"); + } + else + { + // printf("no more data to write, exitting\n"); + break; + } + } + else + break; + } + + for (;;) + { + int pollRc = + poll(fds, nfds, -1); + + if(pollRc == -1) + { + int wasErrno = errno; + + if(wasErrno == EINTR) + { + continue; + } + + perror("FastOS_UNIX_IPCHelper::RunAsync select failure"); + printf("errno = %d\n", wasErrno); + for(unsigned int i = 0; i < nfds; i++) + { + if ((fds[i].events & POLLIN) != 0) + printf("Read %d\n", fds[i].fd); + if ((fds[i].events & POLLOUT) != 0) + printf("Write %d\n", fds[i].fd); + } + exit(1); + } + else + break; + } + + _app->ProcessLock(); + bool woken = SavePollArray(fds, nfds); + // Do actual IO (based on file descriptor sets and buffer contents) + PerformAsyncIO(); + _app->ProcessUnlock(); + PerformAsyncIPCIO(); + + // Did someone want to wake us up from the poll() call? + if (woken) { + char dummy; + read(_wakeupPipe[0], &dummy, 1); + } + } + free(fds); + + delete this; + // printf("IPCHelper exits\n"); +} + + +bool FastOS_UNIX_IPCHelper:: +SendMessage (FastOS_UNIX_Process *xproc, const void *buffer, + int length) +{ + bool rc = false; + + FastOS_RingBuffer *ipcBuffer; + + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc != NULL ? + xproc->GetDescriptorHandle(FastOS_UNIX_Process::TYPE_IPC) : + _appParentIPCDescriptor; + ipcBuffer = desc._writeBuffer.get(); + + if(ipcBuffer != NULL) { + ipcBuffer->Lock(); + + if(ipcBuffer->GetWriteSpace() >= int((length + sizeof(int)))) { + memcpy(ipcBuffer->GetWritePtr(), &length, sizeof(int)); + ipcBuffer->Produce(sizeof(int)); + memcpy(ipcBuffer->GetWritePtr(), buffer, length); + ipcBuffer->Produce(length); + + NotifyProcessListChange(); + rc = true; + } + ipcBuffer->Unlock(); + } + return rc; +} + +void FastOS_UNIX_IPCHelper::NotifyProcessListChange () +{ + char dummy = static_cast<char>(1); + write(_wakeupPipe[1], &dummy, 1); +} + +void FastOS_UNIX_IPCHelper::Exit () +{ + _lock.Lock(); + _exitFlag = true; + NotifyProcessListChange(); + _lock.Unlock(); +} + +void FastOS_UNIX_IPCHelper::AddProcess (FastOS_UNIX_Process *xproc) +{ + bool newStream = false; + for(int type=0; type < int(FastOS_UNIX_Process::TYPE_READCOUNT); type++) + { + FastOS_UNIX_Process::DescriptorType type_ = + FastOS_UNIX_Process::DescriptorType(type); + FastOS_UNIX_Process::DescriptorHandle &desc = + xproc->GetDescriptorHandle(type_); + + if (desc._fd != -1) + { + newStream = true; + SetBlocking(desc._fd, false); + } + } + if(newStream) + NotifyProcessListChange(); +} + +void FastOS_UNIX_IPCHelper::RemoveProcess (FastOS_UNIX_Process *xproc) +{ + (void)xproc; + + FastOS_BoolCond closeWait; + + closeWait.SetBusy(); + xproc->_closing = &closeWait; + + NotifyProcessListChange(); + + closeWait.WaitBusy(); +} + +void FastOS_UNIX_IPCHelper::DeliverMessages (FastOS_RingBuffer *buffer) +{ + if(buffer == NULL) + return; + + buffer->Lock(); + + unsigned int readSpace; + while((readSpace = buffer->GetReadSpace()) > sizeof(int)) + { + FastOS_RingBufferData *bufferData = buffer->GetData(); + + if((readSpace - sizeof(int)) >= bufferData->_messageSize) + { + _app->OnReceivedIPCMessage(&bufferData->_buffer[sizeof(int)], + bufferData->_messageSize); + buffer->Consume(sizeof(int) + bufferData->_messageSize); + buffer->RepositionDataAt0(); + } + else + break; + } + + buffer->Unlock(); +} + +void FastOS_UNIX_IPCHelper:: +PipeData (FastOS_UNIX_Process *process, + FastOS_UNIX_Process::DescriptorType type) +{ + FastOS_UNIX_Process::DescriptorHandle &desc = process->GetDescriptorHandle(type); + FastOS_RingBuffer *buffer = desc._readBuffer.get(); + if(buffer == NULL) + return; + + FastOS_ProcessRedirectListener *listener = process->GetListener(type); + if(listener == NULL) + return; + + buffer->Lock(); + + unsigned int readSpace; + while((readSpace = buffer->GetReadSpace()) > 0) { + listener->OnReceiveData(buffer->GetReadPtr(), size_t(readSpace)); + buffer->Consume(readSpace); + } + + if(buffer->GetCloseFlag()) + process->CloseListener(type); + + buffer->Unlock(); +} diff --git a/fastos/src/vespa/fastos/unix_ipc.h b/fastos/src/vespa/fastos/unix_ipc.h new file mode 100644 index 00000000000..678b24002e1 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_ipc.h @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/app.h> +#include <vespa/fastos/process.h> +#include <vespa/fastos/thread.h> +#include <poll.h> + +class FastOS_RingBuffer; + +class FastOS_UNIX_IPCHelper : public FastOS_Runnable +{ +private: + FastOS_UNIX_IPCHelper(const FastOS_UNIX_IPCHelper&); + FastOS_UNIX_IPCHelper& operator=(const FastOS_UNIX_IPCHelper&); + +protected: + FastOS_Mutex _lock; + volatile bool _exitFlag; + FastOS_ApplicationInterface *_app; + + FastOS_UNIX_Process::DescriptorHandle _appParentIPCDescriptor; + + int _wakeupPipe[2]; + + bool DoWrite (FastOS_UNIX_Process::DescriptorHandle &desc); + bool DoRead (FastOS_UNIX_Process::DescriptorHandle &desc); + bool SetBlocking (int fileDescriptor, bool doBlock); + void BuildPollCheck (bool isRead, int filedes, + FastOS_RingBuffer *buffer, bool *check); + void BuildPollArray(pollfd **fds, unsigned int *nfds, + unsigned int *allocnfds); + bool SavePollArray(pollfd *fds, unsigned int nfds); + void PerformAsyncIO (void); + void PerformAsyncIPCIO (void); + void BuildPollChecks(void); + void DeliverMessages (FastOS_RingBuffer *buffer); + void PipeData (FastOS_UNIX_Process *process, + FastOS_UNIX_Process::DescriptorType type); + void RemoveClosingProcesses(void); + +public: + FastOS_UNIX_IPCHelper (FastOS_ApplicationInterface *app, + int appDescriptor); + ~FastOS_UNIX_IPCHelper (); + void Run (FastOS_ThreadInterface *thisThread, void *arg); + bool SendMessage (FastOS_UNIX_Process *xproc, const void *buffer, + int length); + void NotifyProcessListChange (); + void AddProcess (FastOS_UNIX_Process *xproc); + void RemoveProcess (FastOS_UNIX_Process *xproc); + void Exit (); +}; diff --git a/fastos/src/vespa/fastos/unix_mutex.cpp b/fastos/src/vespa/fastos/unix_mutex.cpp new file mode 100644 index 00000000000..f3e882f5af5 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_mutex.cpp @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/mutex.h> + +FastOS_UNIX_Mutex::FastOS_UNIX_Mutex(void) + : FastOS_MutexInterface(), + _mutex() +{ + int error = pthread_mutex_init(&_mutex, NULL); + assert(error == 0); + (void) error; +} + +FastOS_UNIX_Mutex::~FastOS_UNIX_Mutex(void) +{ + pthread_mutex_destroy(&_mutex); +} diff --git a/fastos/src/vespa/fastos/unix_mutex.h b/fastos/src/vespa/fastos/unix_mutex.h new file mode 100644 index 00000000000..ccf8af9d4ea --- /dev/null +++ b/fastos/src/vespa/fastos/unix_mutex.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-02-02 +* @file +* Class definition and implementation for FastOS_UNIX_Mutex +*****************************************************************************/ + + + +#pragma once + + +#include <vespa/fastos/mutex.h> + + +class FastOS_UNIX_Mutex : public FastOS_MutexInterface +{ +private: + FastOS_UNIX_Mutex(const FastOS_UNIX_Mutex &other); + FastOS_UNIX_Mutex & operator = (const FastOS_UNIX_Mutex &other); +protected: + pthread_mutex_t _mutex; + +public: + FastOS_UNIX_Mutex(void); + + ~FastOS_UNIX_Mutex(void); + + bool TryLock (void) { + return pthread_mutex_trylock(&_mutex) == 0; + } + + void Lock(void) { + pthread_mutex_lock(&_mutex); + } + + void Unlock(void) { + pthread_mutex_unlock(&_mutex); + } +}; + + diff --git a/fastos/src/vespa/fastos/unix_process.cpp b/fastos/src/vespa/fastos/unix_process.cpp new file mode 100644 index 00000000000..96074cd07b0 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_process.cpp @@ -0,0 +1,2001 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/autoconf.h> +#include <vespa/fastos/process.h> +#include <vespa/fastos/app.h> +#include <vespa/fastos/unix_ipc.h> +#include <vespa/fastos/time.h> +#include <vector> + +#ifndef AF_LOCAL +#define AF_LOCAL AF_UNIX +#endif + + +extern "C" +{ +extern char **environ; +} + + +#ifndef FASTOS_HAVE_ACCRIGHTSLEN + +#ifndef ALIGN +#define ALIGN(x) (((x) + sizeof(int) - 1) & ~(sizeof(int) - 1)) +#endif + +#ifndef CMSG_SPACE +#define CMSG_SPACE(l) (ALIGN(sizeof(struct cmsghdr)) + ALIGN(l)) +#endif + +#ifndef CMSG_LEN +#define CMSG_LEN(l)(ALIGN(sizeof(struct cmsghdr)) + (l)) +#endif + +#endif + +static pid_t safe_fork (void) +{ + pid_t pid; + int retry = 1; + while((pid = fork()) == -1 && errno == EAGAIN) { + sleep(retry); + if (retry < 4) retry *= 2; + } + return pid; +} + +static int +normalizedWaitStatus(int status) +{ + if (WIFEXITED(status)) + return WEXITSTATUS(status); + else + return (0x80000000 | status); +} + + +// The actual process launched in the proxy process +class FastOS_UNIX_RealProcess +{ +private: + FastOS_UNIX_RealProcess(const FastOS_UNIX_RealProcess&); + FastOS_UNIX_RealProcess& operator=(const FastOS_UNIX_RealProcess&); + +public: + enum + { + STREAM_STDIN = (1 << 0), + STREAM_STDOUT = (1 << 1), + STREAM_STDERR = (1 << 2), + EXEC_SHELL = (1 << 3) + }; + +private: + pid_t _pid; + bool _died; + bool _terse; // Set if using direct fork (bypassing proxy process) + int _streamMask; + + int _stdinDes[2]; + int _stdoutDes[2]; + int _stderrDes[2]; + int _ipcSockPair[2]; + int _handshakeDes[2]; + std::string _runDir; + std::string _stdoutRedirName; + std::string _stderrRedirName; + const char *_path; + std::vector<char> _pathProgBuf; + + void CloseDescriptor(int fd); + void CloseAndResetDescriptor(int *fd); + void CloseDescriptors(void); + +public: + void SetRunDir(const char * runDir) { _runDir = runDir; } + int GetIPCDescriptor(void) const { return _ipcSockPair[0]; } + int GetStdinDescriptor(void) const { return _stdinDes[1]; } + int GetStdoutDescriptor(void) const { return _stdoutDes[0]; } + int GetStderrDescriptor(void) const { return _stderrDes[0]; } + int HandoverIPCDescriptor(void) { + int ret = _ipcSockPair[0]; + _ipcSockPair[0] = -1; + return ret; + } + + int HandoverStdinDescriptor(void) { + int ret = _stdinDes[1]; + _stdinDes[1] = -1; + return ret; + } + + int HandoverStdoutDescriptor(void) { + int ret = _stdoutDes[0]; + _stdoutDes[0] = -1; + return ret; + } + + int HandoverStderrDescriptor(void) { + int ret = _stderrDes[0]; + _stderrDes[0] = -1; + return ret; + } + + void CloseIPCDescriptor(void); + void CloseStdinDescriptor(void); + void CloseStdoutDescriptor(void); + void CloseStderrDescriptor(void); + + FastOS_UNIX_RealProcess *_prev, *_next; + + FastOS_UNIX_RealProcess (int streamMask); + ~FastOS_UNIX_RealProcess(void); + pid_t GetProcessID(void) const { return _pid; } + + bool IsStdinPiped(void) const { + return (_streamMask & STREAM_STDIN ) != 0; + } + + bool IsStdoutPiped(void) const { + return (_streamMask & STREAM_STDOUT) != 0; + } + + bool IsStderrPiped(void) const { + return (_streamMask & STREAM_STDERR) != 0; + } + + bool IsUsingShell(void) const { + return (_streamMask & EXEC_SHELL) != 0; + } + + void SetStdoutRedirName(const char *stdoutRedirName) { + _stdoutRedirName = stdoutRedirName; + } + + void SetStderrRedirName(const char *stderrRedirName) { + _stderrRedirName = stderrRedirName; + } + + void PrepareExecVPE (const char *prog); + + void + ExecVPE (const char *prog, + char *const args[], + char *const env[]); + + static bool IsWhiteSpace (char c); + + static const char * + NextArgument (const char *p, + const char **endArg, + int *length = NULL); + + static int CountArguments (const char *commandLine); + + void + RedirOut(const std::string & filename, + int targetfd, + int exitCodeOnFailure); + + bool + ForkAndExec(const char *command, + char **environmentVariables, + FastOS_UNIX_Process *process, + FastOS_UNIX_ProcessStarter *processStarter); + + bool Setup(void); + pid_t GetProcessId(void) const { return _pid; } + void SetTerse(void) { _terse = true; } + ssize_t HandshakeRead(void *buf, size_t len); + void HandshakeWrite(int val); +}; + + +void +FastOS_UNIX_RealProcess::CloseDescriptor(int fd) +{ + close(fd); +} + + +void +FastOS_UNIX_RealProcess::CloseAndResetDescriptor(int *fd) +{ + if (*fd == -1) + return; + CloseDescriptor(*fd); + *fd = -1; +} + + +void +FastOS_UNIX_RealProcess::CloseDescriptors(void) +{ + CloseAndResetDescriptor(&_stdinDes[0]); + CloseAndResetDescriptor(&_stdinDes[1]); + CloseAndResetDescriptor(&_stdoutDes[0]); + CloseAndResetDescriptor(&_stdoutDes[1]); + CloseAndResetDescriptor(&_stderrDes[0]); + CloseAndResetDescriptor(&_stderrDes[1]); + CloseAndResetDescriptor(&_ipcSockPair[0]); + CloseAndResetDescriptor(&_ipcSockPair[1]); + CloseAndResetDescriptor(&_handshakeDes[0]); + CloseAndResetDescriptor(&_handshakeDes[1]); +} + + +void +FastOS_UNIX_RealProcess::CloseIPCDescriptor(void) +{ + CloseAndResetDescriptor(&_ipcSockPair[0]); +} + + +void +FastOS_UNIX_RealProcess::CloseStdinDescriptor(void) +{ + CloseAndResetDescriptor(&_stdinDes[1]); +} + + +void +FastOS_UNIX_RealProcess::CloseStdoutDescriptor(void) +{ + CloseAndResetDescriptor(&_stdoutDes[0]); +} + + +void +FastOS_UNIX_RealProcess::CloseStderrDescriptor(void) +{ + CloseAndResetDescriptor(&_stderrDes[0]); +} + + +FastOS_UNIX_RealProcess::FastOS_UNIX_RealProcess(int streamMask) + : _pid(-1), + _died(false), + _terse(false), + _streamMask(streamMask), + _runDir(), + _stdoutRedirName(), + _stderrRedirName(), + _path(NULL), + _pathProgBuf(), + _prev(NULL), + _next(NULL) +{ + _stdinDes[0] = _stdinDes[1] = -1; + _stdoutDes[0] = _stdoutDes[1] = -1; + _stderrDes[0] = _stderrDes[1] = -1; + _ipcSockPair[0] = _ipcSockPair[1] = -1; + _handshakeDes[0] = _handshakeDes[1] = -1; +} + + +FastOS_UNIX_RealProcess::~FastOS_UNIX_RealProcess(void) +{ + CloseDescriptors(); +} + + +void +FastOS_UNIX_RealProcess::PrepareExecVPE(const char *prog) +{ + const char *path = NULL; + + char defaultPath[] = ":/usr/ucb:/bin:/usr/bin"; + + if (strchr(prog, '/') != NULL) { + path = ""; + } else { + path = getenv("PATH"); + if (path == NULL) path = defaultPath; + } + _path = path; + _pathProgBuf.resize(strlen(prog) + 1 + strlen(path) + 1); +} + + +void +FastOS_UNIX_RealProcess::ExecVPE (const char *prog, + char *const args[], + char *const env[]) +{ + + char *fullPath = &_pathProgBuf[0]; + const char *path = _path; + + for(;;) + { + char *p; + for (p = fullPath; (*path != '\0') && (*path != ':'); path++) + *p++ = *path; + + if (p > fullPath) *p++ = '/'; + + strcpy(p, prog); + // fprintf(stdout, "Attempting execve [%s]\n", fullPath); + // fflush(stdout); + execve(fullPath, args, env); + + if ((errno == ENOEXEC) || + (errno == ENOMEM) || + (errno == E2BIG) || + (errno == ETXTBSY)) + break; + + if (*path == '\0') break; + path++; + } +} + + +bool +FastOS_UNIX_RealProcess::IsWhiteSpace (char c) +{ + return (c == ' ' || c == '\t'); +} + + +const char * +FastOS_UNIX_RealProcess::NextArgument (const char *p, + const char **endArg, + int *length) +{ + while(*p != '\0') + { + if (!IsWhiteSpace(*p)) { + char quoteChar = '\0'; + if ((*p == '\'') || (*p == '"')) { + quoteChar = *p; + p++; + } + + const char *nextArg = p; + + // Find the end of the argument. + for(;;) + { + if (*p == '\0') { + if (length != NULL) + *length = p - nextArg; + break; + } + + if (quoteChar != '\0') { + if (*p == quoteChar) { + if (length != NULL) + *length = p - nextArg; + p++; + break; + } + } + else + { + if (IsWhiteSpace(*p)) { + if (length != NULL) + *length = p - nextArg; + break; + } + } + p++; + } + + *endArg = p; + return nextArg; + } + p++; + } + return NULL; +} + + +int +FastOS_UNIX_RealProcess::CountArguments (const char *commandLine) +{ + int numArgs = 0; + const char *nextArg = commandLine; + while(NextArgument(nextArg, &nextArg)) + numArgs++; + + return numArgs; +} + + +void +FastOS_UNIX_RealProcess::RedirOut(const std::string & filename, + int targetfd, + int exitCodeOnFailure) +{ + if (filename.empty() || filename[0] != '>') + return; + + int newfd; + if (filename[1] == '>') { + newfd = open(&filename[2], + O_WRONLY | O_CREAT | O_APPEND, + 0666); + if (newfd < 0) { + if (!_terse) { + fprintf(stderr, + "ERROR: Could not open %s for append: %s\n", + &filename[2], + strerror(errno)); + fflush(stderr); + } + _exit(exitCodeOnFailure); + } + } else { + newfd = open(&filename[1], + O_WRONLY | O_CREAT | O_TRUNC, + 0666); + if (newfd < 0) { + if (!_terse) { + fprintf(stderr, + "ERROR: Could not open %s for write: %s\n", + &filename[1], + strerror(errno)); + fflush(stderr); + } + _exit(exitCodeOnFailure); + } + } + if (newfd != targetfd) { + dup2(newfd, targetfd); + CloseDescriptor(newfd); + } +} + + +bool +FastOS_UNIX_RealProcess:: +ForkAndExec(const char *command, + char **environmentVariables, + FastOS_UNIX_Process *process, + FastOS_UNIX_ProcessStarter *processStarter) +{ + bool rc = false; + + pid_t starterPid = getpid(); + pid_t starterPPid = getppid(); + + sprintf(environmentVariables[0], "%s=%d,%d,%d", + "FASTOS_IPC_PARENT", + int(starterPid), int(starterPPid), _ipcSockPair[1]); + + + int numArguments = 0; + char **execArgs = NULL; + + if (!IsUsingShell()) { + numArguments = CountArguments(command); + if (numArguments > 0) { + execArgs = new char *[numArguments + 1]; + const char *nextArg = command; + + for(int i=0; ; i++) { + int length; + const char *arg = NextArgument(nextArg, &nextArg, + &length); + + if (arg == NULL) { + // printf("ARG NULL\n"); + execArgs[i] = NULL; + break; + } + // printf("argLen = %d\n", length); + execArgs[i] = new char[length + 1]; + memcpy(execArgs[i], arg, length); + execArgs[i][length] = '\0'; + // printf("arg %d: [%s]\n", i, execArgs[i]); + } + PrepareExecVPE(execArgs[0]); + } + } + if (process == NULL) { + processStarter->CloseProxyDescs(IsStdinPiped() ? _stdinDes[0] : -1, + IsStdoutPiped() ? _stdoutDes[1] : -1, + IsStderrPiped() ? _stderrDes[1] : -1, + _ipcSockPair[1], + _handshakeDes[0], + _handshakeDes[1]); + } + _pid = safe_fork(); + if (_pid == static_cast<pid_t>(0)) { + // Fork success, child side. + if (IsStdinPiped() && _stdinDes[0] != STDIN_FILENO) { + dup2(_stdinDes[0], STDIN_FILENO); + CloseDescriptor(_stdinDes[0]); + } + _stdinDes[0] = -1; + if (IsStdoutPiped() && _stdoutDes[1] != STDOUT_FILENO) { + dup2(_stdoutDes[1], STDOUT_FILENO); + CloseDescriptor(_stdoutDes[1]); + } + _stdoutDes[1] = -1; + if (IsStderrPiped() && _stderrDes[1] != STDERR_FILENO) { + dup2(_stderrDes[1], STDERR_FILENO); + CloseDescriptor(_stderrDes[1]); + } + _stderrDes[1] = -1; + // FIX! Check error codes for dup2, and do _exit(127) if trouble + + if ( ! _runDir.empty()) { + if (chdir(_runDir.c_str())) { + if (!_terse) { + fprintf(stderr, + "ERROR: Could not chdir to %s: %s\n", + _runDir.c_str(), + strerror(errno)); + fflush(stderr); + } + _exit(126); + } + } + RedirOut(_stdoutRedirName.c_str(), STDOUT_FILENO, 124); + RedirOut(_stderrRedirName.c_str(), STDERR_FILENO, 125); + + CloseDescriptor(_handshakeDes[0]); + _handshakeDes[0] = -1; + if (process != NULL) { + if (!process->GetKeepOpenFilesIfDirectChild()) { + int fdlimit = sysconf(_SC_OPEN_MAX); + // Close everything else + // printf("fdlimit = %d\n", fdlimit); + for(int fd = STDERR_FILENO + 1; fd < fdlimit; fd++) + { + if (fd != _ipcSockPair[1] && + fd != _handshakeDes[1]) + CloseDescriptor(fd); + } + } else { + // Close only other endpoints of pipes. + process->CloseDescriptorsDirectChild(); + } + } else { + processStarter->CloseProxiedChildDescs(); + } + if (fcntl(_handshakeDes[1], F_SETFD, FD_CLOEXEC) != 0) _exit(127); + + HandshakeWrite(0); + + // printf("exev(p)e [%s]\n", command); + if (IsUsingShell()) { + const char *shExecArgs[4]; + + shExecArgs[0] = "sh"; + shExecArgs[1] = "-c"; + shExecArgs[2] = command; + shExecArgs[3] = NULL; + execve("/bin/sh", + const_cast<char *const *> + (reinterpret_cast<const char *const *> + (shExecArgs)), + environmentVariables); + int error = errno; + if (!_terse) { + fprintf(stderr, + "ERROR: Could not execv /bin/sh -c '%s': %s\n", + command, + strerror(error)); + fflush(stderr); + } + HandshakeWrite(error); + } + else + { + if (numArguments > 0) { + // printf("Command: [%s]\n", execArgs[0]); + ExecVPE(execArgs[0], + static_cast<char *const *>(execArgs), + environmentVariables); + int error = errno; + if (!_terse) { + fprintf(stderr, + "ERROR: Could not execve %s with " + "path search: %s\n", + execArgs[0], + strerror(error)); + fflush(stderr); + } + HandshakeWrite(error); + } + } + _exit(127); // If execve fails, we'll get it here + } + else if(_pid != static_cast<pid_t>(-1)) + { + /* Fork success, parent side */ + + // Close unused file descriptors + if (IsStdinPiped()) { + CloseAndResetDescriptor(&_stdinDes[0]); + } + if (IsStdoutPiped()) { + CloseAndResetDescriptor(&_stdoutDes[1]); + } + if (IsStderrPiped()) { + CloseAndResetDescriptor(&_stderrDes[1]); + } + + CloseAndResetDescriptor(&_ipcSockPair[1]); + + CloseAndResetDescriptor(&_handshakeDes[1]); + + int flags = fcntl(_handshakeDes[0], F_GETFL, 0); + if (flags != -1) { + flags &= ~O_NONBLOCK; + fcntl(_handshakeDes[0], F_SETFL, flags); + } + int phase1res = 0; + ssize_t rgot = HandshakeRead(&phase1res, sizeof(int)); + bool wasError = false; + int error = 0; + if (static_cast<size_t>(rgot) != sizeof(int)) wasError = true; + else if (phase1res != 0) { + wasError = true; + error = phase1res; + } else { + int phase2res = 0; + rgot = HandshakeRead(&phase2res, sizeof(int)); + if (rgot >= 1) { + if (static_cast<size_t>(rgot) >= sizeof(int)) + error = phase2res; + wasError = true; + } + } + + if (wasError) { + int status = 0; + CloseDescriptors(); + pid_t wpid = waitpid(_pid, &status, 0); + if (wpid <= 0) { + fprintf(stderr, "ERROR: Could not start process %s\n", command); + } else if (WIFEXITED(status)) { + status = WEXITSTATUS(status); + switch (status) { + case 124: + if ( ! _stdoutRedirName.empty() && + _stdoutRedirName[0] == '>') { + if (_stdoutRedirName[1] == '>') + fprintf(stderr, "ERROR: Could not open %s for append", &_stdoutRedirName[2]); + else + fprintf(stderr, "ERROR: Could not open %s for write", &_stdoutRedirName[1]); + } + break; + case 125: + if ( ! _stderrRedirName.empty() && + _stderrRedirName[0] == '>') { + if (_stderrRedirName[1] == '>') + fprintf(stderr, "ERROR: Could not open %s for append", &_stderrRedirName[2]); + else + fprintf(stderr, "ERROR: Could not open %s for write", &_stderrRedirName[1]); + } + break; + case 126: + if ( ! _runDir.empty()) { + fprintf(stderr, "ERROR: Could not chdir to %s\n", _runDir.c_str()); + } + break; + case 127: + if (error != 0) { + char errorBuf[100]; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "ERROR: Could not execve %s: %s\n", command, errorString); + } else + fprintf(stderr, "ERROR: Could not execve %s\n", command); + break; + default: + fprintf(stderr, "ERROR: Could not start process %s\n", command); + break; + } + } else { + fprintf(stderr, "ERROR: Could not start process %s\n", command); + } + fflush(stderr); + } else { + rc = true; + } + } + if (execArgs != NULL) { + char **arg = execArgs; + while (*arg != NULL) { + delete [] *arg; + arg++; + } + delete [] execArgs; + } + + return rc; +} + + +void +FastOS_UNIX_RealProcess::HandshakeWrite(int val) +{ + if (_handshakeDes[1] == -1) + return; + const void *wbuf = &val; + size_t residue = sizeof(val); + for (;;) { + /* + * XXX: Might need to use syscall(SYS_write....) to avoid + * thread library interference. + */ + ssize_t wgot = write(_handshakeDes[1], wbuf, residue); + if (wgot < 0 && errno == EINTR) + continue; + if (wgot <= 0) + break; + wbuf = static_cast<const char *>(wbuf) + wgot; + residue -= wgot; + if (residue == 0) + break; + } +} + + +ssize_t +FastOS_UNIX_RealProcess::HandshakeRead(void *buf, size_t len) +{ + if (_handshakeDes[0] == -1) + return 0; + size_t residue = len; + ssize_t rgot = 0; + void *rbuf = buf; + for (;;) { + rgot = read(_handshakeDes[0], rbuf, residue); + if (rgot < 0 && errno == EINTR) + continue; + if (rgot <= 0) + break; + rbuf = static_cast<char *>(rbuf) + rgot; + residue -= rgot; + if (residue == 0) + break; + } + return (residue == len) ? rgot : len - residue; +} + + +bool +FastOS_UNIX_RealProcess::Setup(void) +{ + bool rc = true; + + if (IsStdinPiped()) rc = rc && (pipe(_stdinDes) == 0); + if (IsStdoutPiped()) rc = rc && (pipe(_stdoutDes) == 0); + if (IsStderrPiped()) rc = rc && (pipe(_stderrDes) == 0); + if (!IsUsingShell()) rc = rc && (socketpair(AF_LOCAL, SOCK_STREAM, + 0, _ipcSockPair) == 0); + rc = rc && (pipe(_handshakeDes) == 0); + return rc; +} + + +FastOS_UNIX_Process:: +FastOS_UNIX_Process (const char *cmdLine, bool pipeStdin, + FastOS_ProcessRedirectListener *stdoutListener, + FastOS_ProcessRedirectListener *stderrListener, + int bufferSize) : + FastOS_ProcessInterface(cmdLine, pipeStdin, stdoutListener, + stderrListener, bufferSize), + _pid(0), + _died(false), + _directChild(true), + _keepOpenFilesIfDirectChild(false), + _returnCode(-1), + _descriptor(), + _runDir(), + _stdoutRedirName(), + _stderrRedirName(), + _killed(false), + _closing(NULL) +{ + _descriptor[TYPE_IPC]._readBuffer.reset(new FastOS_RingBuffer(bufferSize)); + _descriptor[TYPE_IPC]._writeBuffer.reset(new FastOS_RingBuffer(bufferSize)); + + if (stdoutListener != NULL) + _descriptor[TYPE_STDOUT]._readBuffer.reset(new FastOS_RingBuffer(bufferSize)); + if (stderrListener != NULL) + _descriptor[TYPE_STDERR]._readBuffer.reset(new FastOS_RingBuffer(bufferSize)); + + _app->ProcessLock(); + _app->AddChildProcess(this); + _app->ProcessUnlock(); + + // App::AddToIPCComm() is performed when the process is started +} + +FastOS_UNIX_Process::~FastOS_UNIX_Process () +{ + Kill(); // Kill if not dead or detached. + + if ((GetDescriptorHandle(TYPE_IPC)._fd != -1) || + (GetDescriptorHandle(TYPE_STDOUT)._fd != -1) || + (GetDescriptorHandle(TYPE_STDERR)._fd != -1)) + { + // Let the IPC helper flush write queues and remove us from the + // process list before we disappear. + static_cast<FastOS_UNIX_Application *>(_app)->RemoveFromIPCComm(this); + } else { + // No IPC descriptor, do it ourselves + _app->ProcessLock(); + _app->RemoveChildProcess(this); + _app->ProcessUnlock(); + } + + for(int i=0; i<int(TYPE_COUNT); i++) { + _descriptor[i]._readBuffer.reset(); + _descriptor[i]._writeBuffer.reset(); + CloseDescriptor(DescriptorType(i)); + } + + CloseListener(TYPE_STDOUT); + CloseListener(TYPE_STDERR); +} + + +void +FastOS_UNIX_Process::SetRunDir(const char *runDir) +{ + _runDir = runDir ? runDir : ""; +} + + +void +FastOS_UNIX_Process::SetStdoutRedirName(const char *stdoutRedirName) +{ + _stdoutRedirName = stdoutRedirName ? stdoutRedirName : ""; +} + + +void +FastOS_UNIX_Process::SetStderrRedirName(const char *stderrRedirName) { + _stderrRedirName = stderrRedirName ? stderrRedirName : ""; +} + + +bool FastOS_UNIX_Process::CreateInternal (bool useShell) +{ + return GetProcessStarter()->CreateProcess(this, useShell, + _pipeStdin, + _stdoutListener != NULL, + _stderrListener != NULL); +} + +bool FastOS_UNIX_Process::WriteStdin (const void *data, size_t length) +{ + bool rc = false; + DescriptorHandle &desc = GetDescriptorHandle(TYPE_STDIN); + + if (desc._fd != -1) { + if (data == NULL) { + CloseDescriptor(TYPE_STDIN); + rc = true; + } + else + { + int writerc = write(desc._fd, data, length); + if (writerc < int(length)) + CloseDescriptor(TYPE_STDIN); + else + rc = true; + } + } + + return rc; +} + +bool FastOS_UNIX_Process::Signal(int sig) +{ + bool rc = false; + pid_t pid; + + _app->ProcessLock(); + pid = GetProcessId(); + if (pid == 0) { + /* Do nothing */ + } else if (GetDeathFlag()) { + rc = true; // The process is no longer around. + } else if (kill(pid, sig) == 0) { + if (sig == SIGKILL) + _killed = true; + rc = true; + } + _app->ProcessUnlock(); + return rc; +} + +bool FastOS_UNIX_Process::Kill () +{ + return Signal(SIGKILL); +} + +bool FastOS_UNIX_Process::WrapperKill () +{ + return Signal(SIGTERM); +} + +bool FastOS_UNIX_Process::InternalWait (int *returnCode, + int timeOutSeconds, + bool *pollStillRunning) +{ + bool rc = GetProcessStarter()->Wait(this, timeOutSeconds, + pollStillRunning); + if (rc) { + if (_killed) + *returnCode = KILL_EXITCODE; + else + *returnCode = _returnCode; + } + + return rc; +} + +bool FastOS_UNIX_Process::Wait (int *returnCode, int timeOutSeconds) +{ + return InternalWait(returnCode, timeOutSeconds, NULL); +} + +bool FastOS_UNIX_Process::PollWait (int *returnCode, bool *stillRunning) +{ + return InternalWait(returnCode, -1, stillRunning); +} + + +bool FastOS_UNIX_Process::Detach(void) +{ + return GetProcessStarter()->Detach(this); +} + + +bool FastOS_UNIX_Process::SendIPCMessage (const void *data, + size_t length) +{ + if (_descriptor[TYPE_IPC]._fd != -1) { + return static_cast<FastOS_UNIX_Application *> + (_app)->SendIPCMessage(this, data, length); + } + return false; +} + +int FastOS_UNIX_Process::BuildStreamMask (bool useShell) +{ + int streamMask = 0; + + if (_pipeStdin) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDIN; + if (_stdoutListener) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDOUT; + if (_stderrListener) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDERR; + if (useShell) streamMask |= FastOS_UNIX_RealProcess::EXEC_SHELL; + + return streamMask; +} + +void FastOS_UNIX_ProcessStarter:: +ReadBytes(int fd, void *buffer, int bytes) +{ + + uint8_t *writePtr = static_cast<uint8_t *>(buffer); + int remaining = bytes; + + while(remaining > 0) + { + int bytesRead; + do + { + bytesRead = read(fd, writePtr, remaining); + } while(bytesRead < 0 && (errno == EINTR)); + + if (bytesRead < 0) { + //perror("FATAL: FastOS_UNIX_ProcessStarter read"); + exit(1); + } + else if(bytesRead == 0) + { + //fprintf(stderr, "FATAL: FastOS_UNIX_RealProcessStart read 0\n"); + exit(1); + } + + writePtr += bytesRead; + remaining -= bytesRead; + } +} + +void FastOS_UNIX_ProcessStarter:: +WriteBytes(int fd, const void *buffer, int bytes, bool ignoreFailure) +{ + const uint8_t *readPtr = static_cast<const uint8_t *>(buffer); + int remaining = bytes; + + while(remaining > 0) + { + int bytesWritten; + do { + bytesWritten = write(fd, readPtr, remaining); + } while (bytesWritten < 0 && (errno == EINTR)); + + if (bytesWritten < 0) { + if (ignoreFailure) + return; + //perror("FATAL: FastOS_UNIX_ProcessStarter write"); + exit(1); + } else if (bytesWritten == 0) { + if (ignoreFailure) + return; + //fprintf(stderr, "FATAL: FastOS_UNIX_RealProcessStart write 0\n"); + exit(1); + } + + readPtr += bytesWritten; + remaining -= bytesWritten; + } +} + +int FastOS_UNIX_ProcessStarter::ReadInt (int fd) +{ + int intStorage; + ReadBytes(fd, &intStorage, sizeof(int)); + return intStorage; +} + +void FastOS_UNIX_ProcessStarter::WriteInt (int fd, int integer, + bool ignoreFailure) +{ + int intStorage = integer; + WriteBytes(fd, &intStorage, sizeof(int), ignoreFailure); +} + +void FastOS_UNIX_ProcessStarter:: +AddChildProcess (FastOS_UNIX_RealProcess *node) +{ + node->_prev = NULL; + node->_next = _processList; + + if (_processList != NULL) + _processList->_prev = node; + _processList = node; +} + +void FastOS_UNIX_ProcessStarter:: +RemoveChildProcess (FastOS_UNIX_RealProcess *node) +{ + if (node->_prev) + node->_prev->_next = node->_next; + else + _processList = node->_next; + + if (node->_next) { + node->_next->_prev = node->_prev; + node->_next = NULL; + } + + if (node->_prev != NULL) + node->_prev = NULL; +} + +bool FastOS_UNIX_ProcessStarter::SendFileDescriptor (int fd) +{ + // printf("SENDFILEDESCRIPTOR\n"); + bool rc = false; + + struct msghdr msg; + struct iovec iov; + + memset(&msg, 0, sizeof(msg)); + memset(&iov, 0, sizeof(iov)); + +#ifndef FASTOS_HAVE_ACCRIGHTSLEN + union + { + struct cmsghdr cm; + char control[CMSG_SPACE(sizeof(int))]; + } control_un; + struct cmsghdr *cmptr; + + memset(&control_un, 0, sizeof(control_un)); + msg.msg_control = control_un.control; + msg.msg_controllen = sizeof(control_un.control); + + cmptr = CMSG_FIRSTHDR(&msg); + cmptr->cmsg_len = CMSG_LEN(sizeof(int)); + cmptr->cmsg_level = SOL_SOCKET; + cmptr->cmsg_type = SCM_RIGHTS; + memcpy(CMSG_DATA(cmptr), &fd, sizeof(int)); +#else + msg.msg_accrights = static_cast<caddr_t>(&fd); + msg.msg_accrightslen = sizeof(int); +#endif + + msg.msg_name = NULL; + msg.msg_namelen = 0; + + char dummyData = '\0'; + iov.iov_base = &dummyData; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + int sendmsgrc = sendmsg(_starterSocketDescr, &msg, 0); + // printf("sendmsg = %d\n", sendmsgrc); + if (sendmsgrc < 0) + perror("sendmsg"); + else + rc = true; + + return rc; +} + +void FastOS_UNIX_ProcessStarter::StarterDoWait () +{ + // printf("WAIT FOR PROCESSES\n"); + + pid_t pid; + int status; + + pid_t deadProcesses[MAX_PROCESSES_PER_WAIT]; + int returnCodes[MAX_PROCESSES_PER_WAIT]; + int numDeadProcesses = 0; + + while((pid = waitpid(-1, &status, WNOHANG)) > 0) + { + // printf("Child %d has died\n", pid); + bool foundProcess = false; + + FastOS_UNIX_RealProcess *process, *next; + for(process = FastOS_UNIX_ProcessStarter::_processList; + process != NULL; process = next) + { + + // Need to do this here since we are deleting entries + next = process->_next; + + if (process->GetProcessId() == pid) { + foundProcess = true; + RemoveChildProcess(process); + delete process; + break; + } + } + + if (!foundProcess && !_hasDetachedProcess) + printf("*** Strange... We don't know about pid %d\n", int(pid)); + + if (!foundProcess) + continue; /* Don't report death of detached processes */ + + deadProcesses[numDeadProcesses] = pid; + + returnCodes[numDeadProcesses] = normalizedWaitStatus(status); + + numDeadProcesses++; + if (numDeadProcesses == MAX_PROCESSES_PER_WAIT) + break; + } + + WriteBytes(_starterSocket, &numDeadProcesses, sizeof(int)); + for(int i=0; i<numDeadProcesses; i++) + { + WriteBytes(_starterSocket, &deadProcesses[i], sizeof(pid_t)); + WriteBytes(_starterSocket, &returnCodes[i], sizeof(int)); + } +} + +void FastOS_UNIX_ProcessStarter::StarterDoDetachProcess () +{ + // printf("DETACH A PROCESS\n"); + + pid_t dpid; + pid_t pid; + int status; + + int returnCode; + + ReadBytes(_starterSocket, &dpid, sizeof(pid_t)); + + bool foundProcess = false; + + pid = waitpid(dpid, &status, WNOHANG); + + FastOS_UNIX_RealProcess *process, *next; + for(process = FastOS_UNIX_ProcessStarter::_processList; + process != NULL; process = next) { + + // Need to do this here since we are deleting entries + next = process->_next; + + if (process->GetProcessId() == dpid) { + foundProcess = true; + _hasDetachedProcess = true; + RemoveChildProcess(process); + delete process; + break; + } + } + + if (foundProcess) { + if (pid == dpid) { + if (WIFEXITED(status)) + returnCode = WEXITSTATUS(status); + else + returnCode = -1; + } else + returnCode = FastOS_ProcessInterface::DETACH_EXITCODE; + } else + returnCode = FastOS_ProcessInterface::NOTFOUND_EXITCODE; + + WriteBytes(_starterSocket, &returnCode, sizeof(int)); +} + +void FastOS_UNIX_ProcessStarter::StarterDoCreateProcess () +{ + int stringLength = ReadInt(_starterSocket); + char cmdLine[stringLength]; + + ReadBytes(_starterSocket, cmdLine, stringLength); + int streamMask = ReadInt(_starterSocket); + char **environmentVariables = ReceiveEnvironmentVariables(); + + FastOS_UNIX_RealProcess *process = new FastOS_UNIX_RealProcess(streamMask); + + bool rc=false; + + int runDirLength = ReadInt(_starterSocket); + + if (runDirLength > 0) { + char runDir[runDirLength]; + ReadBytes(_starterSocket, runDir, runDirLength); + process->SetRunDir(runDir); + } + + int stdoutRedirNameLen = ReadInt(_starterSocket); + if (stdoutRedirNameLen > 0) { + char stdoutRedirName[stdoutRedirNameLen]; + ReadBytes(_starterSocket, stdoutRedirName, stdoutRedirNameLen); + process->SetStdoutRedirName(stdoutRedirName); + } + + int stderrRedirNameLen = ReadInt(_starterSocket); + if (stderrRedirNameLen > 0) { + char stderrRedirName[stderrRedirNameLen]; + ReadBytes(_starterSocket, stderrRedirName, stderrRedirNameLen); + process->SetStderrRedirName(stderrRedirName); + } + + if (process->Setup()) { + WriteInt(_starterSocket, CODE_SUCCESS); + + // Send IPC descriptor if the shell is not used + if (process->IsUsingShell()) + rc = true; + else { + if (SendFileDescriptor(process->GetIPCDescriptor())) { + process->CloseIPCDescriptor(); + WriteInt(_starterSocket, CODE_SUCCESS); + if (ReadInt(_starterSocket) == CODE_SUCCESS) + rc = true; + } else { + WriteInt(_starterSocket, CODE_FAILURE); + } + } + + if (rc) { + rc = false; + + if (!process->IsStdinPiped()) { + rc = true; + } else { + if (SendFileDescriptor(process->GetStdinDescriptor())) { + process->CloseStdinDescriptor(); + WriteInt(_starterSocket, CODE_SUCCESS); + if (ReadInt(_starterSocket) == CODE_SUCCESS) + rc = true; + } else { + WriteInt(_starterSocket, CODE_FAILURE); + } + } + } + + if (rc) { + rc = false; + + if (!process->IsStdoutPiped()) { + rc = true; + } else { + if (SendFileDescriptor(process->GetStdoutDescriptor())) { + process->CloseStdoutDescriptor(); + WriteInt(_starterSocket, CODE_SUCCESS); + if (ReadInt(_starterSocket) == CODE_SUCCESS) { + rc = true; + } + } else { + WriteInt(_starterSocket, CODE_FAILURE); + } + } + } + + if (rc) { + rc = false; + + if (!process->IsStderrPiped()) { + rc = true; + } else { + if (SendFileDescriptor(process->GetStderrDescriptor())) { + process->CloseStderrDescriptor(); + WriteInt(_starterSocket, CODE_SUCCESS); + if (ReadInt(_starterSocket) == CODE_SUCCESS) + rc = true; + } else { + WriteInt(_starterSocket, CODE_FAILURE); + } + } + } + + if (rc) { + rc = false; + + pid_t processId = -1; + if (process->ForkAndExec(cmdLine, + environmentVariables, + NULL, + this)) + { + processId = process->GetProcessID(); + AddChildProcess(process); + rc = true; + } + WriteBytes(_starterSocket, &processId, sizeof(pid_t)); + } + } else { + WriteInt(_starterSocket, CODE_FAILURE); + } + + + if (!rc) delete process; + + char **pe = environmentVariables; + while(*pe != NULL) { + delete [] *pe++; + } + delete [] environmentVariables; +} + +void FastOS_UNIX_ProcessStarter::Run () +{ + for(;;) + { + // Receive commands from main process + int command = ReadInt(_starterSocket); + + switch(command) + { + case CODE_WAIT: StarterDoWait(); break; + case CODE_DETACH: StarterDoDetachProcess(); break; + case CODE_NEWPROCESS: StarterDoCreateProcess(); break; + case CODE_EXIT: _exit(2); + } + } +} + +bool FastOS_UNIX_ProcessStarter::CreateSocketPairs () +{ + bool rc = false; + int fileDescriptors[2]; + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, fileDescriptors) == 0) { + _starterSocket = fileDescriptors[0]; + _mainSocket = fileDescriptors[1]; + + // We want to use a separate pair of sockets for passing + // file descriptors, as errors sending file descriptors + // shouldn't intefere with handshaking of the main <-> starter + // process protocol. + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, fileDescriptors) == 0) { + _starterSocketDescr = fileDescriptors[0]; + _mainSocketDescr = fileDescriptors[1]; + rc = true; + } else { + char errorBuf[100]; + int error = errno; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "socketpair() failed: %s\n", errorString); + } + } else { + char errorBuf[100]; + int error = errno; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "socketpair() failed: %s\n", errorString); + } + return rc; +} + +bool FastOS_UNIX_ProcessStarter::Start () +{ + bool rc = false; + + if (CreateSocketPairs()) { + pid_t pid = safe_fork(); + if (pid != -1) { + if (pid == 0) { // Child + close(_mainSocket); // Close unused end of pipes + close(_mainSocketDescr); + _mainSocket = -1; + _mainSocketDescr = -1; + Run(); // Never returns + } else { // Parent + _pid = pid; + close(_starterSocket); // Close unused end of pipes + close(_starterSocketDescr); + _starterSocket = -1; + _starterSocketDescr = -1; + rc = true; + } + } else { + char errorBuf[100]; + int error = errno; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "could not fork(): %s\n", errorString); + } + } else { + char errorBuf[100]; + int error = errno; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "could not CreateSocketPairs: %s\n", + errorString); + } + return rc; +} + +void FastOS_UNIX_ProcessStarter::Stop () +{ + // Ignore failure (if it already died of SIGINT, etc..) + WriteInt(_mainSocket, CODE_EXIT, true); + + int result = -1; + waitpid(_pid, &result, 0); +} + +int FastOS_UNIX_ProcessStarter::ReadFileDescriptor () +{ + struct msghdr msg; + struct iovec iov; + +#ifndef FASTOS_HAVE_ACCRIGHTSLEN + union + { + struct cmsghdr cm; + char control[CMSG_SPACE(sizeof(int))]; + } control_un; + struct cmsghdr *cmptr; + + msg.msg_control = control_un.control; + msg.msg_controllen = sizeof(control_un.control); +#else + int newfd; + msg.msg_accrights = static_cast<caddr_t>(&newfd); + msg.msg_accrightslen = sizeof(int); +#endif + + msg.msg_name = NULL; + msg.msg_namelen = 0; + + char dummyData = '\0'; + iov.iov_base = &dummyData; + iov.iov_len = 1; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + int recvmsgrc = recvmsg(_mainSocketDescr, &msg, 0); + + if (recvmsgrc <= 0) { + perror("recvmsg"); + return recvmsgrc; + } + // printf("recvmsgrc = %d\n", recvmsgrc); + +#ifndef FASTOS_HAVE_ACCRIGHTSLEN + if ((cmptr = CMSG_FIRSTHDR(&msg)) != NULL && + cmptr->cmsg_len == CMSG_LEN(sizeof(int))) + { + if (cmptr->cmsg_level != SOL_SOCKET) + perror("FastOS_UNIX_ProcessStarter: control level != SOL_SOCKET"); + if (cmptr->cmsg_type != SCM_RIGHTS) + perror("FastOS_UNIX_ProcessStarter: control type != SCM_RIGHTS"); + int fileDescriptor; + memcpy(&fileDescriptor, CMSG_DATA(cmptr), sizeof(int)); + return fileDescriptor; + } + else + return -1; +#else + if (msg.msg_accrightslen == sizeof(int)) + return newfd; + else + return -1; +#endif +} + +bool FastOS_UNIX_ProcessStarter:: +ReceiveFileDescriptor (bool use, FastOS_UNIX_Process::DescriptorType type, + FastOS_UNIX_Process *xproc) +{ + bool rc = false; + if (!use) + rc = true; + else { + int sendFileDescResult = ReadInt(_mainSocket); + + if (sendFileDescResult == CODE_SUCCESS) { + int fd = ReadFileDescriptor(); + + // printf("Got file descriptor %d, type %d\n", fd, type); + + if (fd > 0) { + xproc->SetDescriptor(type, fd); + rc = true; + WriteInt(_mainSocket, CODE_SUCCESS); + } + else + WriteInt(_mainSocket, CODE_FAILURE); + } + } + + return rc; +} + +char ** FastOS_UNIX_ProcessStarter::ReceiveEnvironmentVariables () +{ + int numEnvVars = ReadInt(_starterSocket); + + // printf("Receiving %d environment variables\n", numEnvVars); + char **myEnvironment = new char *[numEnvVars + 2]; + + // Reserve the first entry for the IPC parent variable + myEnvironment[0] = new char [1024]; + + int fillIndex=1; + for(int i=0; i<numEnvVars; i++) + { + int envBytes = ReadInt(_starterSocket); + myEnvironment[fillIndex] = new char [envBytes]; + ReadBytes(_starterSocket, myEnvironment[fillIndex], envBytes); + // printf("Received [%s]\n", myEnvironment[fillIndex]); + + if (strlen(myEnvironment[fillIndex]) == 0 || + strncmp(myEnvironment[fillIndex], "FASTOS_IPC_PARENT=", 18) == 0) + delete [] myEnvironment[fillIndex]; + else + fillIndex++; + } + myEnvironment[fillIndex] = NULL; + + return myEnvironment; +} + + +void FastOS_UNIX_ProcessStarter::SendEnvironmentVariables () +{ + int numEnvVars = 0; + char **pe = environ; + + while(*pe++ != NULL) + numEnvVars++; + + WriteInt(_mainSocket, numEnvVars); + + // In case someone deletes environment variables at the same time + bool gotNull = false; + + pe = environ; + for(int i=0; i<numEnvVars; i++) + { + const char *envString = ""; + if (!gotNull) { + envString = *pe++; + if (envString == NULL) { + envString = ""; + gotNull = true; + } + } + + int envBytes = strlen(envString) + 1; + WriteInt(_mainSocket, envBytes); + WriteBytes(_mainSocket, envString, envBytes); + } +} + + +void +FastOS_UNIX_ProcessStarter:: +CloseProxiedChildDescs(void) +{ + if (_starterSocket >= 0) close(_starterSocket); + if (_starterSocketDescr >= 0) close(_starterSocketDescr); +} + + +void +FastOS_UNIX_ProcessStarter:: +CloseProxyDescs(int stdinPipedDes, + int stdoutPipedDes, + int stderrPipedDes, + int ipcDes, + int handshakeDes0, + int handshakeDes1) +{ + return; + if (_closedProxyProcessFiles) + return; + int fdlimit = sysconf(_SC_OPEN_MAX); + for(int fd = STDERR_FILENO + 1; fd < fdlimit; fd++) + { + if (fd != stdinPipedDes && + fd != stdoutPipedDes && + fd != stderrPipedDes && + fd != ipcDes && + fd != handshakeDes0 && + fd != handshakeDes1 && + fd != _starterSocket && + fd != _starterSocketDescr) + close(fd); + } + _closedProxyProcessFiles = true; +} + +char ** +FastOS_UNIX_ProcessStarter:: +CopyEnvironmentVariables(void) +{ + char **env = environ; + while (*env != NULL) + env++; + int numEnvVars = env - environ; + char **newEnv = new char *[numEnvVars + 2]; + newEnv[0] = new char[1024]; + + int fillIdx = 1; + env = environ; + while (*env != NULL) { + size_t len = strlen(*env); + if (len > 0 && + strncmp(*env, "FASTOS_IPC_PARENT=", 18) != 0) { + newEnv[fillIdx] = new char[len + 1]; + memcpy(newEnv[fillIdx], *env, len + 1); + fillIdx++; + } + env++; + } + newEnv[fillIdx] = NULL; + return newEnv; +} + + +void +FastOS_UNIX_ProcessStarter:: +FreeEnvironmentVariables(char **env) +{ + char **p = env; + while (*p != NULL) { + delete [] *p; + p++; + } + delete [] env; +} + + +bool +FastOS_UNIX_ProcessStarter:: +CreateProcess (FastOS_UNIX_Process *process, + bool useShell, + bool pipeStdin, + bool pipeStdout, + bool pipeStderr) +{ + bool rc = false; + + const char *cmdLine = process->GetCommandLine(); + + process->_app->ProcessLock(); + + if (process->GetDirectChild()) { + _hasDirectChildren = true; + FastOS_UNIX_RealProcess *rprocess = + new FastOS_UNIX_RealProcess(process->BuildStreamMask(useShell)); + const char *runDir = process->GetRunDir(); + if (runDir != NULL) { + rprocess->SetRunDir(runDir); // Handover + } + const char *stdoutRedirName = process->GetStdoutRedirName(); + if (stdoutRedirName != NULL) { + rprocess->SetStdoutRedirName(stdoutRedirName); + } + const char *stderrRedirName = process->GetStderrRedirName(); + if (stderrRedirName != NULL) { + rprocess->SetStderrRedirName(stderrRedirName); + } + char **env = CopyEnvironmentVariables(); + rprocess->SetTerse(); + rprocess->Setup(); + if (!useShell) + process->SetDescriptor(FastOS_UNIX_Process::TYPE_IPC, + rprocess->HandoverIPCDescriptor()); + if (pipeStdin) + process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDIN, + rprocess->HandoverStdinDescriptor()); + if (pipeStdout) + process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDOUT, + rprocess->HandoverStdoutDescriptor()); + if (pipeStderr) + process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDERR, + rprocess->HandoverStderrDescriptor()); + pid_t processId = -1; + if (rprocess->ForkAndExec(cmdLine, env, process, this)) { + processId = rprocess->GetProcessID(); + } + if (processId != -1) { + process->SetProcessId(static_cast<unsigned int>(processId)); + if (!useShell || pipeStdout || pipeStderr) + static_cast<FastOS_UNIX_Application *> + (_app)->AddToIPCComm(process); + rc = true; + } else { + fprintf(stderr, + "Forkandexec %s failed\n", + cmdLine); + } + process->_app->ProcessUnlock(); + delete rprocess; + FreeEnvironmentVariables(env); + return rc; + } + + _hasProxiedChildren = true; + int cmdLineLength = strlen(cmdLine) + 1; + WriteInt(_mainSocket, FastOS_UNIX_ProcessStarter::CODE_NEWPROCESS); + WriteInt(_mainSocket, cmdLineLength); + WriteBytes(_mainSocket, cmdLine, cmdLineLength); + WriteInt(_mainSocket, process->BuildStreamMask(useShell)); + + SendEnvironmentVariables(); + + const char *runDir = process->GetRunDir(); + if (runDir != NULL) { + int runDirLength = strlen(runDir) + 1; + WriteInt(_mainSocket, runDirLength); + WriteBytes(_mainSocket, runDir, runDirLength); + } else + WriteInt(_mainSocket, 0); + + const char *stdoutRedirName = process->GetStdoutRedirName(); + if (stdoutRedirName != NULL) { + int stdoutRedirNameLength = strlen(stdoutRedirName) + 1; + WriteInt(_mainSocket, stdoutRedirNameLength); + WriteBytes(_mainSocket, stdoutRedirName, stdoutRedirNameLength); + } else + WriteInt(_mainSocket, 0); + + const char *stderrRedirName = process->GetStderrRedirName(); + if (stderrRedirName != NULL) { + int stderrRedirNameLength = strlen(stderrRedirName) + 1; + WriteInt(_mainSocket, stderrRedirNameLength); + WriteBytes(_mainSocket, stderrRedirName, stderrRedirNameLength); + } else + WriteInt(_mainSocket, 0); + + + // Check return code of process Setup + if (ReadInt(_mainSocket) == CODE_SUCCESS) { + if (ReceiveFileDescriptor(!useShell, + FastOS_UNIX_Process::TYPE_IPC, + process)) + { + if (ReceiveFileDescriptor(pipeStdin, + FastOS_UNIX_Process::TYPE_STDIN, + process)) + { + if (ReceiveFileDescriptor(pipeStdout, + FastOS_UNIX_Process::TYPE_STDOUT, + process)) + { + if (ReceiveFileDescriptor(pipeStderr, + FastOS_UNIX_Process::TYPE_STDERR, + process)) + { + pid_t processId; + ReadBytes(_mainSocket, &processId, sizeof(pid_t)); + // printf("Received pid = %d\n", int(processId)); + + if (processId != -1) { + process-> + SetProcessId(static_cast<unsigned int>(processId)); + + if (!useShell || pipeStdout || pipeStderr) + static_cast<FastOS_UNIX_Application *> + (_app)->AddToIPCComm(process); + + rc = true; + } + } + } + } + } + } + process->_app->ProcessUnlock(); + + return rc; +} + + +void +FastOS_UNIX_ProcessStarter::PollReapDirectChildren(void) +{ + int status; + pid_t pid; + + for (;;) { + pid = waitpid(-1, &status, WNOHANG); + if (pid <= 0) + break; + + FastOS_ProcessInterface *node; + for(node = _app->GetProcessList(); + node != NULL; node = node->_next) + { + FastOS_UNIX_Process *xproc = + static_cast<FastOS_UNIX_Process *>(node); + + if (xproc->GetProcessId() == static_cast<unsigned int>(pid)) + xproc->DeathNotification(normalizedWaitStatus(status)); + } + } +} + + +void +FastOS_UNIX_ProcessStarter::PollReapProxiedChildren(void) +{ + // Ask our process starter to report dead processes + WriteInt(_mainSocket, FastOS_UNIX_ProcessStarter::CODE_WAIT); + + int numDeadProcesses; + ReadBytes(_mainSocket, &numDeadProcesses, sizeof(int)); + + for(int i=0; i<numDeadProcesses; i++) + { + pid_t deadProcess; + int returnCode; + + ReadBytes(_mainSocket, &deadProcess, sizeof(pid_t)); + ReadBytes(_mainSocket, &returnCode, sizeof(int)); + + FastOS_ProcessInterface *node; + for(node = _app->GetProcessList(); node != NULL; node = node->_next) + { + FastOS_UNIX_Process *xproc = + static_cast<FastOS_UNIX_Process *>(node); + + if (xproc->GetProcessId() == static_cast<unsigned int>(deadProcess)) + { + xproc->DeathNotification(returnCode); + } + } + } +} + + +bool +FastOS_UNIX_ProcessStarter::Wait(FastOS_UNIX_Process *process, + int timeOutSeconds, + bool *pollStillRunning) +{ + bool rc = true; + + bool timeOutKillAttempted = false; + + FastOS_Time startTime; + startTime.SetNow(); + + if (pollStillRunning != NULL) + *pollStillRunning = true; + + for (;;) { + process->_app->ProcessLock(); + + if (_hasDirectChildren) PollReapDirectChildren(); + + if (_hasProxiedChildren) PollReapProxiedChildren(); + + process->_app->ProcessUnlock(); + + if (process->GetDeathFlag()) { + if (pollStillRunning != NULL) + *pollStillRunning = false; + break; + } + + // printf("wasn't dead yet (%d), sleeping\n", + // process->GetProcessId()); + + if (pollStillRunning != NULL) + break; + + if ((timeOutSeconds != -1) && !timeOutKillAttempted) { + FastOS_Time waitTime; + waitTime.SetNow(); + + waitTime -= startTime; + + if (waitTime.MilliSecs() >= (timeOutSeconds * 1000)) { + process->Kill(); + timeOutKillAttempted = true; + } + } + + + // Sleep 100 ms + FastOS_Thread::Sleep(100); + } + + return rc; +} + +bool FastOS_UNIX_ProcessStarter::Detach(FastOS_UNIX_Process *process) +{ + bool rc = true; + pid_t pid; + + process->_app->ProcessLock(); + + pid = process->GetProcessId(); + + if (pid == 0) { + process->_app->ProcessUnlock(); + return false; // Cannot detach nonstarted process. + } + if (process->GetDeathFlag()) { + process->_app->ProcessUnlock(); + return true; + } + + if (process->GetDirectChild()) { + int status = 0; + _hasDetachedProcess = true; + pid_t rpid = waitpid(pid, &status, WNOHANG); + if (rpid > 0) { + status = normalizedWaitStatus(status); + } else if (rpid < 0) + status = FastOS_ProcessInterface::NOTFOUND_EXITCODE; + else + status = FastOS_ProcessInterface::DETACH_EXITCODE; + process->DeathNotification(status); + } else { + // Ask our process starter to detach process + WriteInt(_mainSocket, FastOS_UNIX_ProcessStarter::CODE_DETACH); + WriteBytes(_mainSocket, &pid, sizeof(pid_t)); + + int returnCode; + ReadBytes(_mainSocket, &returnCode, sizeof(int)); + process->DeathNotification(returnCode); + } + process->_app->ProcessUnlock(); + return rc; +} + +bool FastOS_UNIX_Process::SetPriority (Priority priority) +{ + int newPriority; + + switch (priority) + { + case PRIORITY_LOWEST: newPriority = 20; break; + case PRIORITY_BELOW_NORMAL: newPriority = 10; break; + case PRIORITY_NORMAL: newPriority = 0; break; + case PRIORITY_ABOVE_NORMAL: newPriority = -10; break; + case PRIORITY_HIGHEST: newPriority = -20; break; + default: + newPriority = 0; + } + + return (setpriority(PRIO_PROCESS, _pid, newPriority) == 0); + +} diff --git a/fastos/src/vespa/fastos/unix_process.h b/fastos/src/vespa/fastos/unix_process.h new file mode 100644 index 00000000000..269920b5a2a --- /dev/null +++ b/fastos/src/vespa/fastos/unix_process.h @@ -0,0 +1,349 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * Class definitions for FastOS_UNIX_Process. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + +#include <vespa/fastos/process.h> +#include <vespa/fastos/app.h> +#include <string> +#include <memory> + +class FastOS_ThreadPool; +class FastOS_UNIX_RealProcess; + +#include <vespa/fastos/ringbuffer.h> + +class FastOS_UNIX_Process : public FastOS_ProcessInterface +{ +private: + FastOS_UNIX_Process(const FastOS_UNIX_Process&); + FastOS_UNIX_Process& operator=(const FastOS_UNIX_Process&); + + unsigned int _pid; + bool _died; + bool _directChild; + bool _keepOpenFilesIfDirectChild; + int _returnCode; +public: + class DescriptorHandle + { + private: + DescriptorHandle(const DescriptorHandle &); + DescriptorHandle& operator=(const DescriptorHandle &); + + public: + int _fd; + bool _wantRead; + bool _wantWrite; + bool _canRead; + bool _canWrite; + int _pollIdx; + std::unique_ptr<FastOS_RingBuffer> _readBuffer; + std::unique_ptr<FastOS_RingBuffer> _writeBuffer; + DescriptorHandle(void) + : _fd(-1), + _wantRead(false), + _wantWrite(false), + _canRead(false), + _canWrite(false), + _pollIdx(-1), + _readBuffer(), + _writeBuffer() + { + } + void CloseHandle(void) + { + _wantRead = false; + _wantWrite = false; + _canRead = false; + _canWrite = false; + _pollIdx = -1; + if (_fd != -1) { + close(_fd); + _fd = -1; + } + if (_readBuffer.get() != NULL) + _readBuffer->Close(); + if (_writeBuffer.get() != NULL) + _writeBuffer->Close(); + } + void CloseHandleDirectChild(void) + { + if (_fd != -1) { + close(_fd); + _fd = -1; + } + } + }; +private: + DescriptorHandle _descriptor[4]; + + std::string _runDir; + std::string _stdoutRedirName; + std::string _stderrRedirName; + bool _killed; + + FastOS_UNIX_ProcessStarter *GetProcessStarter () + { + return static_cast<FastOS_UNIX_Application *>(_app)-> + GetProcessStarter(); + } + + bool InternalWait (int *returnCode, int timeOutSeconds, + bool *pollStillRunning); +public: + enum DescriptorType + { + TYPE_STDOUT, + TYPE_STDERR, + TYPE_IPC, + TYPE_STDIN, + TYPE_COUNT + }; + + enum Constants + { + TYPE_READCOUNT = 3 + }; + FastOS_BoolCond *_closing; + FastOS_ProcessRedirectListener *GetListener (DescriptorType type) + { + if(type == TYPE_STDOUT) + return _stdoutListener; + else if(type == TYPE_STDERR) + return _stderrListener; + + return NULL; + } + + void CloseListener (DescriptorType type) + { + if(type == TYPE_STDOUT) + { + if(_stdoutListener != NULL) + { + _stdoutListener->OnReceiveData(NULL, 0); + _stdoutListener = NULL; + } + } + else if(type == TYPE_STDERR) + { + if(_stderrListener != NULL) + { + _stderrListener->OnReceiveData(NULL, 0); + _stderrListener = NULL; + } + } + } + + FastOS_UNIX_Process (const char *cmdLine, bool pipeStdin = false, + FastOS_ProcessRedirectListener *stdoutListener = NULL, + FastOS_ProcessRedirectListener *stderrListener = NULL, + int bufferSize = 65535); + ~FastOS_UNIX_Process (); + bool CreateInternal (bool useShell); + bool Create () { return CreateInternal(false); } + bool CreateWithShell () { return CreateInternal(true); } + bool WriteStdin (const void *data, size_t length); + bool Signal(int sig); + bool Kill (); + bool WrapperKill (); + bool Wait (int *returnCode, int timeOutSeconds = -1); + bool PollWait (int *returnCode, bool *stillRunning); + bool Detach(void); + void SetProcessId (unsigned int pid) { _pid = pid; } + unsigned int GetProcessId() { return _pid; } + bool SendIPCMessage (const void *data, size_t length); + void DeathNotification (int returnCode) + { + _returnCode = returnCode; + _died = true; + } + bool GetDeathFlag () { return _died; } + int BuildStreamMask (bool useShell); + + void CloseDescriptor (DescriptorType type) + { + _descriptor[type].CloseHandle(); + } + + void CloseDescriptorDirectChild(DescriptorType type) + { + _descriptor[type].CloseHandleDirectChild(); + } + + void CloseDescriptorsDirectChild(void) + { + CloseDescriptorDirectChild(TYPE_STDOUT); + CloseDescriptorDirectChild(TYPE_STDERR); + CloseDescriptorDirectChild(TYPE_IPC); + CloseDescriptorDirectChild(TYPE_STDIN); + } + + void SetDescriptor (DescriptorType type, + int descriptor) + { + _descriptor[type]._fd = descriptor; + } + + DescriptorHandle &GetDescriptorHandle(DescriptorType type) + { + return _descriptor[type]; + } + + FastOS_ProcessRedirectListener *GetStdoutListener () + { + return _stdoutListener; + } + FastOS_ProcessRedirectListener *GetStderrListener () + { + return _stderrListener; + } + bool GetKillFlag () {return _killed; } + virtual bool GetDirectChild(void) const + { + return _directChild; + } + + virtual bool SetDirectChild(void) + { + _directChild = true; + return true; + } + + virtual bool GetKeepOpenFilesIfDirectChild(void) const + { + return _keepOpenFilesIfDirectChild; + } + + virtual bool SetKeepOpenFilesIfDirectChild(void) + { + _keepOpenFilesIfDirectChild = true; + return true; + } + + bool SetPriority (Priority pri); + void SetRunDir(const char *runDir); + void SetStdoutRedirName(const char *stdoutRedirName); + void SetStderrRedirName(const char *stderrRedirName); + const char *GetRunDir(void) const { return _runDir.c_str(); } + const char *GetStdoutRedirName(void) const { return _stdoutRedirName.c_str(); } + const char *GetStderrRedirName(void) const { return _stderrRedirName.c_str(); } +}; + + +class FastOS_UNIX_RealProcess; +class FastOS_UNIX_ProcessStarter +{ +private: + FastOS_UNIX_ProcessStarter(const FastOS_UNIX_ProcessStarter&); + FastOS_UNIX_ProcessStarter& operator=(const FastOS_UNIX_ProcessStarter&); + +public: + + enum Constants + { + CODE_EXIT, + CODE_NEWPROCESS, + CODE_WAIT, + CODE_DETACH, + + CODE_SUCCESS, + CODE_FAILURE, + + MAX_PROCESSES_PER_WAIT = 50, + + CONSTEND + }; + +protected: + FastOS_ApplicationInterface *_app; + static void ReadBytes(int fd, void *buffer, int bytes); + static void WriteBytes(int fd, const void *buffer, + int bytes, bool ignoreFailure = false); + static int ReadInt (int fd); + static void WriteInt (int fd, int integer, bool ignoreFailure = false); + + FastOS_UNIX_RealProcess *_processList; + + pid_t _pid; + int _starterSocket; + int _mainSocket; + int _starterSocketDescr; + int _mainSocketDescr; + bool _hasProxiedChildren; + bool _closedProxyProcessFiles; + bool _hasDetachedProcess; + bool _hasDirectChildren; + + void StarterDoWait (); + void StarterDoDetachProcess (); + void StarterDoCreateProcess (); + + bool SendFileDescriptor (int fd); + int ReadFileDescriptor (); + + char **ReceiveEnvironmentVariables (); + void SendEnvironmentVariables (); + + bool CreateSocketPairs (); + void Run (); + + void AddChildProcess (FastOS_UNIX_RealProcess *node); + void RemoveChildProcess (FastOS_UNIX_RealProcess *node); + + bool ReceiveFileDescriptor (bool use, + FastOS_UNIX_Process::DescriptorType type, + FastOS_UNIX_Process *xproc); + void PollReapProxiedChildren(void); + char **CopyEnvironmentVariables(void); + void FreeEnvironmentVariables(char **env); + void PollReapDirectChildren(void); + +public: + FastOS_UNIX_ProcessStarter (FastOS_ApplicationInterface *app) + : _app(app), + _processList(NULL), + _pid(-1), + _starterSocket(-1), + _mainSocket(-1), + _starterSocketDescr(-1), + _mainSocketDescr(-1), + _hasProxiedChildren(false), + _closedProxyProcessFiles(false), + _hasDetachedProcess(false), + _hasDirectChildren(false) + { + } + + ~FastOS_UNIX_ProcessStarter () + { + if(_starterSocket != -1) + close(_starterSocket); + if(_mainSocket != -1) + close(_mainSocket); + } + + bool Start (); + void Stop (); + void CloseProxiedChildDescs(void); + void CloseProxyDescs(int stdinPipedDes, + int stdoutPipedDes, + int stderrPipedDes, + int ipcDes, + int handshakeDes0, + int handshakeDes1); + + bool CreateProcess (FastOS_UNIX_Process *process, bool useShell, + bool pipeStdin, bool pipeStdout, bool pipeStderr); + bool Wait (FastOS_UNIX_Process *process, int timeOutSeconds, + bool *pollStillRunning); + bool Detach(FastOS_UNIX_Process *process); +}; + + diff --git a/fastos/src/vespa/fastos/unix_socket.cpp b/fastos/src/vespa/fastos/unix_socket.cpp new file mode 100644 index 00000000000..a6cce42bfad --- /dev/null +++ b/fastos/src/vespa/fastos/unix_socket.cpp @@ -0,0 +1,155 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/socket.h> + +FastOS_UNIX_Socket::~FastOS_UNIX_Socket() +{ + FastOS_UNIX_Socket::Close(); +} + +bool FastOS_UNIX_Socket::Close() +{ + bool rc=true; + + if (ValidHandle()) { + CleanupEvents(); + rc = (0 == close(_socketHandle)); + _socketHandle = -1; + } + + return rc; +} + +bool FastOS_UNIX_Socket::Shutdown() +{ + bool rc=true; + + if (ValidHandle()) { + if(_socketEvent != NULL) { + EnableWriteEvent(false); + } + rc = (0 == shutdown(_socketHandle, SHUT_WR)); + } + + return rc; +} + +bool FastOS_UNIX_Socket::SetSoBlocking (bool blockingEnabled) +{ + bool rc=false; + + if (CreateIfNoSocketYet()) { + int flags = fcntl(_socketHandle, F_GETFL, NULL); + + if (flags >= 0) { + if (blockingEnabled) { + flags &= ~O_NONBLOCK; // clear nonblocking + } else { + flags |= O_NONBLOCK; // set nonblocking + } + + if (fcntl(_socketHandle, F_SETFL, flags) >= 0) { + rc = true; + } + } + } + + return rc; +} + +ssize_t FastOS_UNIX_Socket::Write (const void *writeBuffer, size_t bufferSize) +{ + assert(ValidHandle()); + + ssize_t got; + do { + got = ::write(_socketHandle, writeBuffer, bufferSize); + } while (got<0 && errno == EINTR); // caught interrupt; nonBlocking sock.s + + return got; +} + + +ssize_t FastOS_UNIX_Socket::Read (void *readBuffer, size_t bufferSize) +{ + assert(ValidHandle()); + + ssize_t got; + do { + got = ::read(_socketHandle, readBuffer, bufferSize); + } while (got<0 && errno == EINTR); // caught interrupt; nonBlocking sock.s + + return got; +} + + +std::string +FastOS_UNIX_Socket::getHostName(std::string * errorMsg) +{ + char buffer[MAXHOSTNAMELEN]; + if (gethostname(buffer, sizeof(buffer)) < 0) { + if (errorMsg != NULL) { + int errnoCopy = errno; + // Malloc calculations here are not 100% correct (does not compensate + // for printf codes). Allocating a few extra bytes isn't farmful though. + const char errorUname[]="Failed to use gethostname() to get host name: %s"; + char errorBuf[100]; + const char *errorString = strerror_r(errnoCopy, errorBuf, sizeof(errorBuf)); + char *errorMsgC = static_cast<char *> + (malloc(strlen(errorUname) + + strlen(errorString) + 1)); + sprintf(errorMsgC, errorUname, errorString); + *errorMsg = errorMsgC; + free(errorMsgC); + } + } else { + buffer[sizeof(buffer) - 1] = '\0'; + return std::string(buffer); + } + + return std::string(); +} + + +std::string +FastOS_UNIX_Socket::getErrorString(int error) +{ + char errorBuf[100]; + const char *errorString = strerror_r(error, errorBuf, sizeof(errorBuf)); + return std::string(errorString); +} + +bool FastOS_SocketEventObjects::Init (FastOS_SocketEvent *event) +{ + (void)event; + + _wakeUpPipe[0] = -1; + _wakeUpPipe[1] = -1; + + if (pipe(_wakeUpPipe) == 0) { + int flags; + flags = fcntl(_wakeUpPipe[0], F_GETFL, 0); + if (flags != -1) { + flags |= O_NONBLOCK; + fcntl(_wakeUpPipe[0], F_SETFL, flags); + } + flags = fcntl(_wakeUpPipe[1], F_GETFL, 0); + if (flags != -1) { + flags |= O_NONBLOCK; + fcntl(_wakeUpPipe[1], F_SETFL, flags); + } + return true; + } else { + return false; + } +} + +void FastOS_SocketEventObjects::Cleanup () +{ + if(_wakeUpPipe[0] != -1) { + close(_wakeUpPipe[0]); + } + if(_wakeUpPipe[1] != -1) { + close(_wakeUpPipe[1]); + } +} diff --git a/fastos/src/vespa/fastos/unix_socket.h b/fastos/src/vespa/fastos/unix_socket.h new file mode 100644 index 00000000000..e269d926ed6 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_socket.h @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/fastos/socket.h> + +class FastOS_UNIX_Socket : public FastOS_SocketInterface +{ +public: + ~FastOS_UNIX_Socket(); + + bool Close (); + bool Shutdown(); + bool SetSoBlocking (bool blockingEnabled); + ssize_t Read (void *readBuffer, size_t bufferSize) override; + ssize_t Write (const void *writeBuffer, size_t bufferSize) override; + + static std::string getHostName(std::string *errorMsg); + static std::string getHostName() { return getHostName(nullptr); } + static int GetLastError () { return errno; } + static std::string getErrorString(int error); + + enum { + ERR_ALREADY = EALREADY, // New style error codes + ERR_AGAIN = EAGAIN, + ERR_INTR = EINTR, + ERR_ISCONN = EISCONN, + ERR_INPROGRESS = EINPROGRESS, + ERR_WOULDBLOCK = EWOULDBLOCK, + ERR_ADDRNOTAVAIL = EADDRNOTAVAIL, + ERR_MFILE = FASTOS_EMFILE_VERIFIED, + ERR_NFILE = FASTOS_ENFILE_VERIFIED, + ERR_CONNRESET = ECONNRESET, + + ERR_EAGAIN = EAGAIN, // Old style error codes + ERR_EINTR = EINTR, + ERR_EISCONN = EISCONN, + ERR_EINPROGRESS = EINPROGRESS, + ERR_EWOULDBLOCK = EWOULDBLOCK, + ERR_EADDRNOTAVAIL = EADDRNOTAVAIL, + ERR_EMFILE = FASTOS_EMFILE_VERIFIED, + ERR_ENFILE = FASTOS_ENFILE_VERIFIED + }; +}; + + diff --git a/fastos/src/vespa/fastos/unix_thread.cpp b/fastos/src/vespa/fastos/unix_thread.cpp new file mode 100644 index 00000000000..fe92bafb47b --- /dev/null +++ b/fastos/src/vespa/fastos/unix_thread.cpp @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/thread.h> +#include <atomic> +#include <thread> + +namespace { + std::atomic_size_t _G_nextCpuId(0); + volatile size_t _G_maxNumCpus=0; // Non zero means use cpu pinning. +} + +bool FastOS_UNIX_Thread::InitializeClass () +{ + if (getenv("VESPA_PIN_THREAD_TO_CORE") != NULL) { + _G_maxNumCpus = std::thread::hardware_concurrency(); + fprintf(stderr, "Will pin threads to CPU. Using %ld cores\n", _G_maxNumCpus); + if (getenv("VESPA_MAX_CORES") != NULL) { + size_t maxCores = strtoul(getenv("VESPA_MAX_CORES"), NULL, 0); + fprintf(stderr, "Will limit to %ld", maxCores); + if (maxCores < _G_maxNumCpus) { + _G_maxNumCpus = maxCores; + } + } + } + return true; +} + +bool FastOS_UNIX_Thread::CleanupClass () +{ + return true; +} + +bool FastOS_UNIX_Thread::Initialize (int stackSize, int stackGuardSize) +{ + bool rc=false; + + pthread_attr_t attr; + pthread_attr_init(&attr); + + pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); + + pthread_attr_setstacksize(&attr, stackSize); + if (_G_maxNumCpus > 0) { + int cpuid = _G_nextCpuId.fetch_add(1)%_G_maxNumCpus; + cpu_set_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(cpuid, &cpuset); + int retval = pthread_attr_setaffinity_np(&attr, sizeof(cpuset), &cpuset); + if (retval != 0) { + fprintf(stderr, "Pinning FAILURE retval = %d, errno=%d sizeof(cpuset_t)=%ld cpuid(%d)\n", retval, errno, sizeof(cpuset), cpuid); + } + } + + if (stackGuardSize != 0) { + pthread_attr_setguardsize(&attr, stackGuardSize); + } + + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + + rc = (0 == pthread_create(&_handle, &attr, FastOS_ThreadHook, this)); + if (rc) + _handleValid = true; + + pthread_attr_destroy(&attr); + + if (rc && pthread_getschedparam(_handle, &_normal_policy, + &_normal_schedparam) == 0) + { + _schedparams_ok = true; + } + _schedparams_changed = false; + + return rc; +} + +void FastOS_UNIX_Thread::PreEntry () +{ + if (_schedparams_changed) { + _schedparams_changed = false; + SetPriority(FastOS_Thread::PRIORITY_NORMAL); + } +} + +bool FastOS_UNIX_Thread::SetPriority (const Priority priority) +{ + bool rc=false; + + if(_schedparams_ok) + { + struct sched_param schedparam; + + schedparam = _normal_schedparam; + schedparam.sched_priority = (priority + + _normal_schedparam.sched_priority); + + if (pthread_setschedparam(_handle, _normal_policy, + &schedparam) == 0) + { + rc = true; + _schedparams_changed = true; + } + } + + return rc; +} + + +FastOS_UNIX_Thread::~FastOS_UNIX_Thread(void) +{ + void *value; + + // Wait for thread library cleanup to complete. + if (_handleValid) { + value = NULL; + pthread_join(_handle, &value); + } +} + +FastOS_ThreadId FastOS_UNIX_Thread::GetThreadId () +{ + return _handle; +} + +FastOS_ThreadId FastOS_UNIX_Thread::GetCurrentThreadId () +{ + return pthread_self(); +} + +bool FastOS_UNIX_Thread::CompareThreadIds (FastOS_ThreadId a, + FastOS_ThreadId b) +{ + return (pthread_equal(a, b) != 0); +} diff --git a/fastos/src/vespa/fastos/unix_thread.h b/fastos/src/vespa/fastos/unix_thread.h new file mode 100644 index 00000000000..6d51d1857b7 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_thread.h @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-02-02 +* @file +* Class definition for FastOS_UNIX_Thread +*****************************************************************************/ + +#pragma once + +#include <vespa/fastos/thread.h> + +class FastOS_UNIX_Thread : public FastOS_ThreadInterface +{ +private: + FastOS_UNIX_Thread(const FastOS_UNIX_Thread &); + FastOS_UNIX_Thread& operator=(const FastOS_UNIX_Thread &); + +protected: + pthread_t _handle; + struct sched_param _normal_schedparam; + int _normal_policy; + bool _handleValid; + bool _schedparams_ok; + bool _schedparams_changed; + + bool Initialize (int stackSize, int stackGuardSize); + void PreEntry (); + +public: + static bool InitializeClass (); + static bool CleanupClass (); + + FastOS_UNIX_Thread(FastOS_ThreadPool *pool) + : FastOS_ThreadInterface(pool), + _handle(), + _normal_schedparam(), + _normal_policy(0), + _handleValid(false), + _schedparams_ok(false), + _schedparams_changed(false) + {} + + virtual ~FastOS_UNIX_Thread(void); + + static bool Sleep (int ms) + { + bool rc=false; + + if (ms > 0) { + usleep(ms*1000); + rc = true; + } + + return rc; + } + + bool SetPriority (const Priority priority); + FastOS_ThreadId GetThreadId (); + static bool CompareThreadIds (FastOS_ThreadId a, + FastOS_ThreadId b); + static FastOS_ThreadId GetCurrentThreadId (); +}; + + diff --git a/fastos/src/vespa/fastos/unix_time.cpp b/fastos/src/vespa/fastos/unix_time.cpp new file mode 100644 index 00000000000..202ffe6d82a --- /dev/null +++ b/fastos/src/vespa/fastos/unix_time.cpp @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/time.h> + +double +FastOS_UNIX_Time::MicroSecs() const +{ + return ((1000.0 * 1000.0) * _time.tv_sec) + _time.tv_usec; +} + +double +FastOS_UNIX_Time::MilliSecs() const +{ + return 1000.0 * _time.tv_sec + static_cast<double>(_time.tv_usec) / 1000.0; +} + +double +FastOS_UNIX_Time::Secs() const +{ + return _time.tv_sec + static_cast<double>(_time.tv_usec) / 1000000.0; +} + +FastOS_UNIX_Time& +FastOS_UNIX_Time::operator=(const FastOS_UNIX_Time& rhs) +{ + _time.tv_sec = rhs._time.tv_sec; + _time.tv_usec = rhs._time.tv_usec; + return *this; +} + + +FastOS_UNIX_Time& +FastOS_UNIX_Time::operator+=(const FastOS_UNIX_Time &rhs) +{ + struct timeval result; + FASTOS_UNIX_TIMER_ADD(&_time, &rhs._time, &result); + _time = result; + return *this; +} + + +FastOS_UNIX_Time& +FastOS_UNIX_Time::operator-=(const FastOS_UNIX_Time &rhs) +{ + struct timeval result; + FASTOS_UNIX_TIMER_SUB(&_time, &rhs._time, &result); + _time = result; + return *this; +} + +void +FastOS_UNIX_Time::SetMicroSecs(double microsecs) +{ + if (microsecs > 0) { + _time.tv_sec = static_cast<int>(microsecs / (1000 * 1000)); + _time.tv_usec = static_cast<int>(microsecs - (1000.0 * 1000.0) * + _time.tv_sec); + } + else { + _time.tv_sec = - static_cast<int>(- microsecs / (1000 * 1000)); + _time.tv_usec = - static_cast<int>(- microsecs - + (1000.0 * 1000.0) * + (- _time.tv_sec)); + } +} + +void +FastOS_UNIX_Time::SetMilliSecs(double millisecs) +{ + if (millisecs > 0) { + _time.tv_sec = static_cast<int>(millisecs/1000); + _time.tv_usec = static_cast<int>((millisecs - 1000.0 * _time.tv_sec) * + 1000); + } + else { + // some of the "-" may be unnecessary . + // round on positive numbers to make sure conversion to int is ok. + _time.tv_sec = - static_cast<int>(- millisecs / 1000); + _time.tv_usec = - static_cast<int>((- millisecs - 1000.0 * + ( -_time.tv_sec)) * 1000); + } +} + + +void +FastOS_UNIX_Time::SetSecs(double secs) +{ + if (secs > 0) { + _time.tv_sec = static_cast<int>(secs); + _time.tv_usec = static_cast<int>((secs - _time.tv_sec) * 1000000); + } + else { + // some of the "-" may be unnecessary . + // round on positive numbers to make sure conversion to int is ok. + _time.tv_sec = - static_cast<int>(- secs); + _time.tv_usec = - static_cast<int>((- secs - (-_time.tv_sec)) * 1000000); + } +} diff --git a/fastos/src/vespa/fastos/unix_time.h b/fastos/src/vespa/fastos/unix_time.h new file mode 100644 index 00000000000..a0b7054e121 --- /dev/null +++ b/fastos/src/vespa/fastos/unix_time.h @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/time.h> +#include <vespa/fastos/timestamp.h> + + +#define FASTOS_UNIX_TIMER_CMP(tvp, uvp, cmp) \ + (((tvp)->tv_sec == (uvp)->tv_sec)? \ + ((tvp)->tv_usec cmp (uvp)->tv_usec) :\ + ((tvp)->tv_sec cmp (uvp)->tv_sec)) + +#define FASTOS_UNIX_TIMER_SUB(tvp, uvp, vvp) \ + do { \ + if ((tvp)->tv_usec >= (uvp)->tv_usec) {\ + (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;\ + } else {\ + (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec - 1; \ + (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec + 1000000;\ + }\ + } while (0) + +#define FASTOS_UNIX_TIMER_ADD(tvp, uvp, vvp) \ + do { \ + if ((tvp)->tv_usec+(uvp)->tv_usec<1000000) {\ + (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec; \ + (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec;\ + } else {\ + (vvp)->tv_sec = (tvp)->tv_sec + (uvp)->tv_sec + 1; \ + (vvp)->tv_usec = (tvp)->tv_usec + (uvp)->tv_usec - 1000000;\ + }\ + } while (0) + + +class FastOS_UNIX_Time : public FastOS_TimeInterface +{ + /* + * Class for OS independent time to millisecond resolution. + * + */ +private: + timeval _time; + +public: + void SetZero() { _time.tv_sec = 0; _time.tv_usec = 0; } + + FastOS_UNIX_Time () + : _time() + { + SetZero(); + } + + FastOS_UNIX_Time (double s) + : _time() + { + SetMilliSecs(s * 1000.0); + } + + operator fastos::TimeStamp () { + return fastos::TimeStamp(_time); + } + + FastOS_UNIX_Time& operator=(const FastOS_UNIX_Time& rhs); + FastOS_UNIX_Time& operator+=(const FastOS_UNIX_Time& rhs); + FastOS_UNIX_Time& operator-=(const FastOS_UNIX_Time& rhs); + + bool operator>(const FastOS_UNIX_Time rhs) const { + return FASTOS_UNIX_TIMER_CMP(&_time, &rhs._time, >); + } + bool operator<(const FastOS_UNIX_Time rhs) const { + return FASTOS_UNIX_TIMER_CMP(&_time, &rhs._time, <); + } + bool operator>=(const FastOS_UNIX_Time rhs) const { + return FASTOS_UNIX_TIMER_CMP(&_time, &rhs._time, >=); + } + bool operator<=(const FastOS_UNIX_Time rhs) const { + return FASTOS_UNIX_TIMER_CMP(&_time, &rhs._time, <=); + } + bool operator==(const FastOS_UNIX_Time rhs) const { + return FASTOS_UNIX_TIMER_CMP(&_time, &rhs._time, ==); + } + + double MicroSecs() const; + double MilliSecs() const; + double Secs() const; + + void SetMicroSecs(double microsecs); + void SetMilliSecs(double millisecs); + void SetSecs(double secs); + + void SetNow() { gettimeofday(&_time, NULL); } + + long int GetSeconds() const { return _time.tv_sec; } + long int GetMicroSeconds() const { return _time.tv_usec; } +}; + diff --git a/fastos/src/vespa/fastos/version.h b/fastos/src/vespa/fastos/version.h new file mode 100644 index 00000000000..18c0a64184a --- /dev/null +++ b/fastos/src/vespa/fastos/version.h @@ -0,0 +1,2 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#define FASTOS_VERSIONSTRING "fastos 1.9.1 (current)" diff --git a/fastos/src/vespa/fastos/vtag.cpp b/fastos/src/vespa/fastos/vtag.cpp new file mode 100644 index 00000000000..a9f00820224 --- /dev/null +++ b/fastos/src/vespa/fastos/vtag.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdio.h> +#include "vtag.h" + +#ifndef V_TAG +#define V_TAG "NOTAG" +#define V_TAG_TYPE "NOTAG" +#define V_TAG_VALUE "NOTAG" +#define V_TAG_DATE "NOTAG" +#define V_TAG_SYSTEM "NOTAG" +#define V_TAG_SYSTEM_REV "NOTAG" +#define V_TAG_BUILDER "NOTAG" +#endif + +namespace fastos { + +char VersionTag[] = V_TAG; +char VersionTagDate[] = V_TAG_DATE; +char VersionTagSystem[] = V_TAG_SYSTEM; +char VersionTagSystemRev[] = V_TAG_SYSTEM_REV; +char VersionTagBuilder[] = V_TAG_BUILDER; + +} // namespace fastos diff --git a/fastos/src/vespa/fastos/vtag.h b/fastos/src/vespa/fastos/vtag.h new file mode 100644 index 00000000000..ee5c065ef1c --- /dev/null +++ b/fastos/src/vespa/fastos/vtag.h @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace fastos { + +extern char VersionTag[]; +extern char VersionTagType[]; +extern char VersionTagValue[]; +extern char VersionTagDate[]; +extern char VersionTagSystem[]; +extern char VersionTagSystemRev[]; +extern char VersionTagBuilder[]; + +} // namespace fastos + |