A performance-oriented Lisp-like language where I can have my cake, and eat it (too)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

592 lines
16 KiB

Fixed Windows bugs due to comptime, cache file * My compile-time functions change broke Windows because Windows requires you link the import library in order to call functions in other DLLs. I added ImportLibraryPath and ImportLibraries as link inputs. These may be merged with the other libraries inputs, once I determine how best to do that (shouldn't be too hard) * Fixed bug where additional references were being created to compile-time functions for no reason, and causing crashes. Context now holds which reference is being resolved * Relocate .pdb files to facilitate parallel compilation. I was encountering an error with multiple processes writing to the same PDB, vc140.pdb. Now, each object will explicitly specify its pdb location * Delete pdbs before building. This only needs to happen when /DEBUG:FASTLINK is specified * Moved Build argument converter into Build.cpp for shared use. This may be how I solve the very painful argument conversion stuff. Much cleanup is needed to make this sustainable * Do not write Cache file if nothing to write. This prevents empty file error when tokenizing cache * Added CAKELISP_API to functions I was using in tests * Use Build_RunTests.bat for a more exhaustive testing * Changed how compile-time functions are declared in order to facilitate DLL importing/exporting. I'm not certain the C linkage is necessary yet * Improved empty file error to print file name * test/BuildHelpers.cake runs Cakelisp itself as a test executable, to make it easier to test cross-platform
2 months ago
Fixed Windows bugs due to comptime, cache file * My compile-time functions change broke Windows because Windows requires you link the import library in order to call functions in other DLLs. I added ImportLibraryPath and ImportLibraries as link inputs. These may be merged with the other libraries inputs, once I determine how best to do that (shouldn't be too hard) * Fixed bug where additional references were being created to compile-time functions for no reason, and causing crashes. Context now holds which reference is being resolved * Relocate .pdb files to facilitate parallel compilation. I was encountering an error with multiple processes writing to the same PDB, vc140.pdb. Now, each object will explicitly specify its pdb location * Delete pdbs before building. This only needs to happen when /DEBUG:FASTLINK is specified * Moved Build argument converter into Build.cpp for shared use. This may be how I solve the very painful argument conversion stuff. Much cleanup is needed to make this sustainable * Do not write Cache file if nothing to write. This prevents empty file error when tokenizing cache * Added CAKELISP_API to functions I was using in tests * Use Build_RunTests.bat for a more exhaustive testing * Changed how compile-time functions are declared in order to facilitate DLL importing/exporting. I'm not certain the C linkage is necessary yet * Improved empty file error to print file name * test/BuildHelpers.cake runs Cakelisp itself as a test executable, to make it easier to test cross-platform
2 months ago
  1. #include "RunProcess.hpp"
  2. #include <stdio.h>
  3. #include <vector>
  4. #ifdef UNIX
  5. #include <string.h>
  6. #include <sys/types.h> // pid
  7. #include <sys/wait.h> // waitpid
  8. #include <unistd.h> // exec, fork
  9. #elif WINDOWS
  10. #define WIN32_LEAN_AND_MEAN
  11. #include <windows.h>
  12. // _tprintf() Remove me!
  13. #include <tchar.h>
  14. #else
  15. #error Platform support is needed for running subprocesses
  16. #endif
  17. #include "Logging.hpp"
  18. #include "Utilities.hpp"
  19. #ifdef UNIX
  20. typedef pid_t ProcessId;
  21. #else
  22. typedef int ProcessId;
  23. #endif
  24. struct Subprocess
  25. {
  26. int* statusOut;
  27. #ifdef UNIX
  28. ProcessId processId;
  29. int pipeReadFileDescriptor;
  30. #elif WINDOWS
  31. PROCESS_INFORMATION* processInfo;
  32. // HANDLE hChildStd_IN_Wr; // Not used
  33. HANDLE hChildStd_OUT_Rd;
  34. #endif
  35. std::string command;
  36. };
  37. static std::vector<Subprocess> s_subprocesses;
  38. // Never returns, if success
  39. void systemExecute(const char* fileToExecute, char** arguments)
  40. {
  41. #ifdef UNIX
  42. // pid_t pid;
  43. execvp(fileToExecute, arguments);
  44. perror("RunProcess execvp() error: ");
  45. Logf("Failed to execute %s\n", fileToExecute);
  46. #endif
  47. }
  48. void subprocessReceiveStdOut(const char* processOutputBuffer)
  49. {
  50. Logf("%s", processOutputBuffer);
  51. }
  52. int runProcess(const RunProcessArguments& arguments, int* statusOut)
  53. {
  54. if (!arguments.arguments)
  55. {
  56. Log("error: runProcess() called with empty arguments. At a minimum, first argument must be "
  57. "executable name\n");
  58. return 1;
  59. }
  60. if (logging.processes)
  61. {
  62. Log("RunProcess command: ");
  63. for (const char** arg = arguments.arguments; *arg != nullptr; ++arg)
  64. {
  65. Logf("%s ", *arg);
  66. }
  67. Log("\n");
  68. }
  69. #ifdef UNIX
  70. int pipeFileDescriptors[2] = {0};
  71. const int PipeRead = 0;
  72. const int PipeWrite = 1;
  73. if (pipe(pipeFileDescriptors) == -1)
  74. {
  75. perror("RunProcess: ");
  76. return 1;
  77. }
  78. pid_t pid = fork();
  79. if (pid == -1)
  80. {
  81. perror("RunProcess fork() error: cannot create child: ");
  82. return 1;
  83. }
  84. // Child
  85. else if (pid == 0)
  86. {
  87. // Redirect std out and err to the pipes instead
  88. if (dup2(pipeFileDescriptors[PipeWrite], STDOUT_FILENO) == -1 ||
  89. dup2(pipeFileDescriptors[PipeWrite], STDERR_FILENO) == -1)
  90. {
  91. perror("RunProcess: ");
  92. return 1;
  93. }
  94. // Only write
  95. close(pipeFileDescriptors[PipeRead]);
  96. char** nonConstArguments = nullptr;
  97. {
  98. int numArgs = 0;
  99. for (const char** arg = arguments.arguments; *arg != nullptr; ++arg)
  100. ++numArgs;
  101. // Add one for null sentinel
  102. nonConstArguments = new char*[numArgs + 1];
  103. int i = 0;
  104. for (const char** arg = arguments.arguments; *arg != nullptr; ++arg)
  105. {
  106. nonConstArguments[i] = StrDuplicate(*arg);
  107. ++i;
  108. }
  109. // Null sentinel
  110. nonConstArguments[numArgs] = nullptr;
  111. }
  112. if (arguments.workingDirectory)
  113. {
  114. if (chdir(arguments.workingDirectory) != 0)
  115. {
  116. Logf("error: RunProcess failed to change directory to '%s'\n",
  117. arguments.workingDirectory);
  118. perror("RunProcess chdir");
  119. goto childProcessFailed;
  120. }
  121. if (logging.processes)
  122. Logf("Set working directory to %s\n", arguments.workingDirectory);
  123. }
  124. systemExecute(arguments.fileToExecute, nonConstArguments);
  125. childProcessFailed:
  126. // This shouldn't happen unless the execution failed or soemthing
  127. for (char** arg = nonConstArguments; *arg != nullptr; ++arg)
  128. delete *arg;
  129. delete[] nonConstArguments;
  130. // A failed child should not flush parent files
  131. _exit(EXIT_FAILURE); /* */
  132. }
  133. // Parent
  134. else
  135. {
  136. // Only read
  137. close(pipeFileDescriptors[PipeWrite]);
  138. if (logging.processes)
  139. Logf("Created child process %d\n", pid);
  140. std::string command = "";
  141. for (const char** arg = arguments.arguments; *arg != nullptr; ++arg)
  142. {
  143. command.append(*arg);
  144. command.append(" ");
  145. }
  146. s_subprocesses.push_back({statusOut, pid, pipeFileDescriptors[PipeRead], command});
  147. }
  148. return 0;
  149. #elif WINDOWS
  150. const char* fileToExecute = arguments.fileToExecute;
  151. // Build a single string with all arguments
  152. char* commandLineString = nullptr;
  153. {
  154. size_t commandLineLength = 0;
  155. bool isFirstArg = true;
  156. for (const char** arg = arguments.arguments; *arg != nullptr; ++arg)
  157. {
  158. if (isFirstArg)
  159. {
  160. commandLineLength += strlen(fileToExecute);
  161. // Room for quotes
  162. commandLineLength += 2;
  163. isFirstArg = false;
  164. }
  165. else
  166. commandLineLength += strlen(*arg);
  167. // Room for space
  168. commandLineLength += 1;
  169. }
  170. commandLineString = (char*)calloc(commandLineLength, sizeof(char));
  171. commandLineString[commandLineLength - 1] = '\0';
  172. char* writeHead = commandLineString;
  173. isFirstArg = true;
  174. for (const char** arg = arguments.arguments; *arg != nullptr; ++arg)
  175. {
  176. // Support executable with spaces in path
  177. if (isFirstArg)
  178. {
  179. isFirstArg = false;
  180. if (!writeCharToBuffer('"', &writeHead, commandLineString, commandLineLength))
  181. {
  182. Log("error: ran out of space to write command\n");
  183. free(commandLineString);
  184. return 1;
  185. }
  186. if (!writeStringToBuffer(fileToExecute, &writeHead, commandLineString,
  187. commandLineLength))
  188. {
  189. Log("error: ran out of space to write command\n");
  190. free(commandLineString);
  191. return 1;
  192. }
  193. if (!writeCharToBuffer('"', &writeHead, commandLineString, commandLineLength))
  194. {
  195. Log("error: ran out of space to write command\n");
  196. free(commandLineString);
  197. return 1;
  198. }
  199. }
  200. else
  201. {
  202. if (!writeStringToBuffer(*arg, &writeHead, commandLineString, commandLineLength))
  203. {
  204. Log("error: ran out of space to write command\n");
  205. free(commandLineString);
  206. return 1;
  207. }
  208. }
  209. if (*(arg + 1) != nullptr)
  210. {
  211. if (!writeCharToBuffer(' ', &writeHead, commandLineString, commandLineLength))
  212. {
  213. Log("error: ran out of space to write command\n");
  214. free(commandLineString);
  215. return 1;
  216. }
  217. }
  218. }
  219. }
  220. if (logging.processes)
  221. Logf("Final command string: %s\n", commandLineString);
  222. // Redirect child process std in/out
  223. HANDLE hChildStd_IN_Rd = nullptr;
  224. HANDLE hChildStd_IN_Wr = nullptr;
  225. HANDLE hChildStd_OUT_Rd = nullptr;
  226. HANDLE hChildStd_OUT_Wr = nullptr;
  227. {
  228. SECURITY_ATTRIBUTES securityAttributes;
  229. // Set the bInheritHandle flag so pipe handles are inherited.
  230. securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES);
  231. securityAttributes.bInheritHandle = TRUE;
  232. securityAttributes.lpSecurityDescriptor = NULL;
  233. // Create a pipe for the child process's STDOUT.
  234. if (!CreatePipe(&hChildStd_OUT_Rd, &hChildStd_OUT_Wr, &securityAttributes, 0))
  235. {
  236. Logf("StdoutRd CreatePipe error %d\n", GetLastError());
  237. free(commandLineString);
  238. return 1;
  239. }
  240. // Ensure the read handle to the pipe for STDOUT is not inherited.
  241. if (!SetHandleInformation(hChildStd_OUT_Rd, HANDLE_FLAG_INHERIT, 0))
  242. {
  243. Logf("Stdout SetHandleInformation error %d\n", GetLastError());
  244. free(commandLineString);
  245. CloseHandle(hChildStd_OUT_Rd);
  246. CloseHandle(hChildStd_OUT_Wr);
  247. return 1;
  248. }
  249. // Create a pipe for the child process's STDIN.
  250. if (!CreatePipe(&hChildStd_IN_Rd, &hChildStd_IN_Wr, &securityAttributes, 0))
  251. {
  252. Logf("Stdin CreatePipe error %d\n", GetLastError());
  253. free(commandLineString);
  254. CloseHandle(hChildStd_OUT_Rd);
  255. CloseHandle(hChildStd_OUT_Wr);
  256. return 1;
  257. }
  258. // Ensure the write handle to the pipe for STDIN is not inherited.
  259. if (!SetHandleInformation(hChildStd_IN_Wr, HANDLE_FLAG_INHERIT, 0))
  260. {
  261. Logf("Stdin SetHandleInformation error %d\n", GetLastError());
  262. free(commandLineString);
  263. CloseHandle(hChildStd_OUT_Rd);
  264. CloseHandle(hChildStd_OUT_Wr);
  265. CloseHandle(hChildStd_IN_Rd);
  266. CloseHandle(hChildStd_IN_Wr);
  267. return 1;
  268. }
  269. }
  270. STARTUPINFO startupInfo;
  271. ZeroMemory(&startupInfo, sizeof(startupInfo));
  272. startupInfo.cb = sizeof(startupInfo);
  273. // This structure specifies the STDIN and STDOUT handles for redirection.
  274. startupInfo.hStdError = hChildStd_OUT_Wr;
  275. startupInfo.hStdOutput = hChildStd_OUT_Wr;
  276. startupInfo.hStdInput = hChildStd_IN_Rd;
  277. startupInfo.dwFlags |= STARTF_USESTDHANDLES;
  278. PROCESS_INFORMATION* processInfo = new PROCESS_INFORMATION;
  279. ZeroMemory(processInfo, sizeof(PROCESS_INFORMATION));
  280. // Start the child process.
  281. if (!CreateProcess(fileToExecute,
  282. commandLineString, // Command line
  283. nullptr, // No security attributes
  284. nullptr, // Thread handle not inheritable
  285. true, // Set handle inheritance to true
  286. 0, // No creation flags
  287. nullptr, // Use parent's environment block
  288. arguments.workingDirectory, // If nullptr, use parent's starting directory
  289. &startupInfo, // Pointer to STARTUPINFO structure
  290. processInfo)) // Pointer to PROCESS_INFORMATION structure
  291. {
  292. CloseHandle(hChildStd_OUT_Rd);
  293. CloseHandle(hChildStd_OUT_Wr);
  294. CloseHandle(hChildStd_IN_Rd);
  295. CloseHandle(hChildStd_IN_Wr);
  296. free(commandLineString);
  297. int errorCode = GetLastError();
  298. if (errorCode == ERROR_FILE_NOT_FOUND)
  299. {
  300. Logf("CreateProcess failed to find file: %s\n", fileToExecute);
  301. }
  302. else if (errorCode == ERROR_PATH_NOT_FOUND)
  303. {
  304. Logf("CreateProcess failed to find path: %s\n", fileToExecute);
  305. }
  306. else
  307. {
  308. Logf("CreateProcess failed: %d\n", errorCode);
  309. }
  310. // LogLastError();
  311. return 1;
  312. }
  313. // Close handles to the stdin and stdout pipes no longer needed by the child process.
  314. // If they are not explicitly closed, there is no way to recognize that the child process ended
  315. CloseHandle(hChildStd_OUT_Wr);
  316. CloseHandle(hChildStd_IN_Rd);
  317. CloseHandle(hChildStd_IN_Wr);
  318. Subprocess newProcess = {0};
  319. newProcess.statusOut = statusOut;
  320. newProcess.processInfo = processInfo;
  321. newProcess.hChildStd_OUT_Rd = hChildStd_OUT_Rd;
  322. newProcess.command = commandLineString;
  323. s_subprocesses.push_back(std::move(newProcess));
  324. free(commandLineString);
  325. return 0;
  326. #endif
  327. return 1;
  328. }
  329. #ifdef WINDOWS
  330. void readProcessPipe(Subprocess& process, SubprocessOnOutputFunc onOutput)
  331. {
  332. HANDLE hParentStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
  333. char buffer[4096] = {0};
  334. bool encounteredError = false;
  335. while (true)
  336. {
  337. DWORD bytesRead = 0;
  338. DWORD bytesWritten = 0;
  339. bool success = ReadFile(process.hChildStd_OUT_Rd, buffer, sizeof(buffer), &bytesRead, NULL);
  340. if (!success || bytesRead == 0)
  341. {
  342. encounteredError = !success;
  343. break;
  344. }
  345. success = WriteFile(hParentStdOut, buffer, bytesRead, &bytesWritten, NULL);
  346. if (!success)
  347. {
  348. encounteredError = true;
  349. break;
  350. }
  351. if (onOutput && bytesRead <= sizeof(buffer))
  352. onOutput(buffer);
  353. }
  354. // This seems to give a lot of false-positives
  355. // if (encounteredError)
  356. // Log("warning: encountered read or write error while receiving sub-process "
  357. // "output\n");
  358. }
  359. #endif
  360. // This function prints all the output for each process in one contiguous block, so that outputs
  361. // between two processes aren't mangled together terribly
  362. void waitForAllProcessesClosed(SubprocessOnOutputFunc onOutput)
  363. {
  364. if (s_subprocesses.empty())
  365. return;
  366. for (Subprocess& process : s_subprocesses)
  367. {
  368. #ifdef UNIX
  369. char processOutputBuffer[1024] = {0};
  370. int numBytesRead =
  371. read(process.pipeReadFileDescriptor, processOutputBuffer, sizeof(processOutputBuffer));
  372. while (numBytesRead > 0)
  373. {
  374. processOutputBuffer[numBytesRead] = '\0';
  375. subprocessReceiveStdOut(processOutputBuffer);
  376. if (onOutput)
  377. onOutput(processOutputBuffer);
  378. numBytesRead = read(process.pipeReadFileDescriptor, processOutputBuffer,
  379. sizeof(processOutputBuffer));
  380. }
  381. close(process.pipeReadFileDescriptor);
  382. waitpid(process.processId, process.statusOut, 0);
  383. // It's pretty useful to see the command which resulted in failure
  384. if (*process.statusOut != 0)
  385. Logf("%s\n", process.command.c_str());
  386. #elif WINDOWS
  387. // We cannot wait indefinitely because the process eventually waits for us to read from the
  388. // output pipe (e.g. its buffer gets full). pollProcessTimeMilliseconds may need to be
  389. // tweaked for better performance; if the buffer is full, the subprocess will wait for as
  390. // long as pollProcessTimeMilliseconds - time taken to fill buffer. Very low wait times will
  391. // mean Cakelisp unnecessarily taking up cycles, so it's a tradeoff.
  392. const int pollProcessTimeMilliseconds = 50;
  393. while (WAIT_TIMEOUT ==
  394. WaitForSingleObject(process.processInfo->hProcess, pollProcessTimeMilliseconds))
  395. readProcessPipe(process, onOutput);
  396. // If the wait was ended but wasn't a timeout, we still need to read out
  397. readProcessPipe(process, onOutput);
  398. DWORD exitCode = 0;
  399. if (!GetExitCodeProcess(process.processInfo->hProcess, &exitCode))
  400. {
  401. Log("error: failed to get exit code for process\n");
  402. exitCode = 1;
  403. }
  404. else if (exitCode != 0)
  405. {
  406. Logf("%s\n", process.command.c_str());
  407. }
  408. *process.statusOut = exitCode;
  409. // Close process, thread, and stdout handles.
  410. CloseHandle(process.processInfo->hProcess);
  411. CloseHandle(process.processInfo->hThread);
  412. CloseHandle(process.hChildStd_OUT_Rd);
  413. #endif
  414. }
  415. s_subprocesses.clear();
  416. }
  417. void PrintProcessArguments(const char** processArguments)
  418. {
  419. for (const char** argument = processArguments; *argument; ++argument)
  420. Logf("%s ", *argument);
  421. Log("\n");
  422. }
  423. static const char* ProcessCommandArgumentTypeToString(ProcessCommandArgumentType type)
  424. {
  425. switch (type)
  426. {
  427. case ProcessCommandArgumentType_None:
  428. return "None";
  429. case ProcessCommandArgumentType_String:
  430. return "String";
  431. case ProcessCommandArgumentType_SourceInput:
  432. return "SourceInput";
  433. case ProcessCommandArgumentType_ObjectOutput:
  434. return "ObjectOutput";
  435. case ProcessCommandArgumentType_DebugSymbolsOutput:
  436. return "DebugSymbolsOutput";
  437. case ProcessCommandArgumentType_ImportLibraryPaths:
  438. return "ImportLibraryPaths";
  439. case ProcessCommandArgumentType_ImportLibraries:
  440. return "ImportLibraries";
  441. case ProcessCommandArgumentType_CakelispHeadersInclude:
  442. return "CakelispHeadersInclude";
  443. case ProcessCommandArgumentType_IncludeSearchDirs:
  444. return "IncludeSearchDirs";
  445. case ProcessCommandArgumentType_AdditionalOptions:
  446. return "AdditionalOptions";
  447. case ProcessCommandArgumentType_PrecompiledHeaderOutput:
  448. return "PrecompiledHeaderOutput";
  449. case ProcessCommandArgumentType_PrecompiledHeaderInclude:
  450. return "PrecompiledHeaderInclude";
  451. case ProcessCommandArgumentType_ObjectInput:
  452. return "ObjectInput";
  453. case ProcessCommandArgumentType_DynamicLibraryOutput:
  454. return "DynamicLibraryOutput";
  455. case ProcessCommandArgumentType_LibrarySearchDirs:
  456. return "LibrarySearchDirs";
  457. case ProcessCommandArgumentType_Libraries:
  458. return "Libraries";
  459. case ProcessCommandArgumentType_LibraryRuntimeSearchDirs:
  460. return "LibraryRuntimeSearchDirs";
  461. case ProcessCommandArgumentType_LinkerArguments:
  462. return "LinkerArguments";
  463. case ProcessCommandArgumentType_ExecutableOutput:
  464. return "ExecutableOutput";
  465. default:
  466. return "Unknown";
  467. }
  468. }
  469. // The array will need to be deleted, but the array members will not
  470. const char** MakeProcessArgumentsFromCommand(const char* fileToExecute,
  471. std::vector<ProcessCommandArgument>& arguments,
  472. const ProcessCommandInput* inputs, int numInputs)
  473. {
  474. std::vector<const char*> argumentsAccumulate;
  475. for (unsigned int i = 0; i < arguments.size(); ++i)
  476. {
  477. ProcessCommandArgument& argument = arguments[i];
  478. if (argument.type == ProcessCommandArgumentType_String)
  479. argumentsAccumulate.push_back(argument.contents.c_str());
  480. else
  481. {
  482. bool found = false;
  483. for (int input = 0; input < numInputs; ++input)
  484. {
  485. if (inputs[input].type == argument.type)
  486. {
  487. for (const char* value : inputs[input].value)
  488. {
  489. if (!value || !value[0])
  490. {
  491. Logf(
  492. "warning: attempted to pass null string to '%s' under argument "
  493. "type %s. It will be ignored\n",
  494. fileToExecute, ProcessCommandArgumentTypeToString(argument.type));
  495. continue;
  496. }
  497. argumentsAccumulate.push_back(value);
  498. }
  499. found = true;
  500. break;
  501. }
  502. }
  503. if (!found)
  504. {
  505. Logf("error: command to %s missing ProcessCommandInput of type %s\n", fileToExecute,
  506. ProcessCommandArgumentTypeToString(argument.type));
  507. return nullptr;
  508. }
  509. }
  510. }
  511. int numUserArguments = argumentsAccumulate.size();
  512. // +1 for file to execute
  513. int numFinalArguments = numUserArguments + 1;
  514. // +1 again for the null terminator
  515. const char** newArguments = (const char**)calloc(sizeof(const char*), numFinalArguments + 1);
  516. newArguments[0] = fileToExecute;
  517. for (int i = 1; i < numFinalArguments; ++i)
  518. newArguments[i] = argumentsAccumulate[i - 1];
  519. newArguments[numFinalArguments] = nullptr;
  520. return newArguments;
  521. }
  522. const int maxProcessesRecommendedSpawned = 8;