Cppcheck
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
preprocessor.cpp
Go to the documentation of this file.
1 /*
2  * Cppcheck - A tool for static C/C++ code analysis
3  * Copyright (C) 2007-2016 Cppcheck team.
4  *
5  * This program is free software: you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation, either version 3 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17  */
18 
19 
20 #include "preprocessor.h"
21 #include "tokenize.h"
22 #include "token.h"
23 #include "path.h"
24 #include "errorlogger.h"
25 #include "settings.h"
26 
27 #include <algorithm>
28 #include <sstream>
29 #include <fstream>
30 #include <cstdlib>
31 #include <cctype>
32 #include <vector>
33 #include <set>
34 #include <stack>
35 
36 /**
37  * Remove heading and trailing whitespaces from the input parameter.
38  * If string is all spaces/tabs, return empty string.
39  * @param s The string to trim.
40  */
41 static std::string trim(const std::string& s)
42 {
43  const std::string::size_type beg = s.find_first_not_of(" \t");
44  if (beg == std::string::npos)
45  return "";
46  const std::string::size_type end = s.find_last_not_of(" \t");
47  return s.substr(beg, end - beg + 1);
48 }
49 
50 Directive::Directive(const std::string &_file, const int _linenr, const std::string &_str):
51  file(_file),
52  linenr(_linenr),
53  str(_str)
54 {
55  // strip C++ comment if there is one
56  std::size_t pos = str.find("//");
57  if (pos != std::string::npos)
58  str.erase(pos);
59  // strip any C comments
60  while ((pos = str.find("/*")) != std::string::npos) {
61  std::size_t end = str.find("*/", pos+2);
62  if (end != std::string::npos) {
63  str.erase(pos, end + 2 - pos);
64  } else { // treat '/*' as '//' if '*/' is missing
65  str.erase(pos);
66  }
67  }
68  str = trim(str);
69 }
70 
73 
74 char Preprocessor::macroChar = char(1);
75 
76 Preprocessor::Preprocessor(Settings& settings, ErrorLogger *errorLogger) : _settings(settings), _errorLogger(errorLogger)
77 {
78 
79 }
80 
81 void Preprocessor::writeError(const std::string &fileName, const unsigned int linenr, ErrorLogger *errorLogger, const std::string &errorType, const std::string &errorText)
82 {
83  if (!errorLogger)
84  return;
85 
86  std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
87  ErrorLogger::ErrorMessage::FileLocation loc(fileName, linenr);
88  locationList.push_back(loc);
89  errorLogger->reportErr(ErrorLogger::ErrorMessage(locationList,
91  errorText,
92  errorType,
93  false));
94 }
95 
96 static unsigned char readChar(std::istream &istr, unsigned int bom)
97 {
98  unsigned char ch = (unsigned char)istr.get();
99 
100  // For UTF-16 encoded files the BOM is 0xfeff/0xfffe. If the
101  // character is non-ASCII character then replace it with 0xff
102  if (bom == 0xfeff || bom == 0xfffe) {
103  const unsigned char ch2 = (unsigned char)istr.get();
104  const int ch16 = (bom == 0xfeff) ? (ch<<8 | ch2) : (ch2<<8 | ch);
105  ch = (unsigned char)((ch16 >= 0x80) ? 0xff : ch16);
106  }
107 
108  // Handling of newlines..
109  if (ch == '\r') {
110  ch = '\n';
111  if (bom == 0 && (char)istr.peek() == '\n')
112  (void)istr.get();
113  else if (bom == 0xfeff || bom == 0xfffe) {
114  int c1 = istr.get();
115  int c2 = istr.get();
116  int ch16 = (bom == 0xfeff) ? (c1<<8 | c2) : (c2<<8 | c1);
117  if (ch16 != '\n') {
118  istr.unget();
119  istr.unget();
120  }
121  }
122  }
123 
124  return ch;
125 }
126 
127 // Concatenates a list of strings, inserting a separator between parts
128 static std::string join(const std::set<std::string>& list, char separator)
129 {
130  std::string s;
131  for (std::set<std::string>::const_iterator it = list.begin(); it != list.end(); ++it) {
132  if (!s.empty())
133  s += separator;
134 
135  s += *it;
136  }
137  return s;
138 }
139 
140 // Removes duplicate string portions separated by the specified separator
141 static std::string unify(const std::string &s, char separator)
142 {
143  std::set<std::string> parts;
144 
145  std::string::size_type prevPos = 0;
146  for (std::string::size_type pos = 0; pos < s.length(); ++pos) {
147  if (s[pos] == separator) {
148  if (pos > prevPos)
149  parts.insert(s.substr(prevPos, pos - prevPos));
150  prevPos = pos + 1;
151  }
152  }
153  if (prevPos < s.length())
154  parts.insert(s.substr(prevPos));
155 
156  return join(parts, separator);
157 }
158 
159 
160 bool Preprocessor::cplusplus(const Settings *settings, const std::string &filename)
161 {
162  const bool undef = settings && settings->userUndefs.find("__cplusplus") != settings->userUndefs.end();
163  const bool cpplang = settings && settings->enforcedLang == Settings::CPP;
164  const bool cppfile = (!settings || settings->enforcedLang == Settings::None) && Path::isCPP(filename);
165  return (!undef && (cpplang || cppfile));
166 }
167 
168 /**
169  * Get cfgmap - a map of macro names and values
170  */
171 static std::map<std::string,std::string> getcfgmap(const std::string &cfg, const Settings *settings, const std::string &filename)
172 {
173  std::map<std::string, std::string> cfgmap;
174 
175  if (!cfg.empty()) {
176  std::string::size_type pos = 0;
177  for (;;) {
178  std::string::size_type pos2 = cfg.find_first_of(";=", pos);
179  if (pos2 == std::string::npos) {
180  cfgmap[cfg.substr(pos)] = "";
181  break;
182  }
183  if (cfg[pos2] == ';') {
184  cfgmap[cfg.substr(pos, pos2-pos)] = "";
185  } else {
186  std::string::size_type pos3 = pos2;
187  pos2 = cfg.find(';', pos2);
188  if (pos2 == std::string::npos) {
189  cfgmap[cfg.substr(pos, pos3-pos)] = cfg.substr(pos3 + 1);
190  break;
191  } else {
192  cfgmap[cfg.substr(pos, pos3-pos)] = cfg.substr(pos3 + 1, pos2 - pos3 - 1);
193  }
194  }
195  pos = pos2 + 1;
196  }
197  }
198 
199  if (cfgmap.find("__cplusplus") == cfgmap.end() && Preprocessor::cplusplus(settings,filename))
200  cfgmap["__cplusplus"] = "1";
201 
202  return cfgmap;
203 }
204 
205 
206 /** Just read the code into a string. Perform simple cleanup of the code */
207 std::string Preprocessor::read(std::istream &istr, const std::string &filename)
208 {
209  // The UTF-16 BOM is 0xfffe or 0xfeff.
210  unsigned int bom = 0;
211  if (istr.peek() >= 0xfe) {
212  bom = ((unsigned char)istr.get() << 8);
213  if (istr.peek() >= 0xfe)
214  bom |= (unsigned char)istr.get();
215  else
216  bom = 0; // allowed boms are 0/0xfffe/0xfeff
217  }
218 
219  if (_settings.terminated())
220  return "";
221 
223  return readpreprocessor(istr,bom);
224 
225  // ------------------------------------------------------------------------------------------
226  //
227  // handling <backslash><newline>
228  // when this is encountered the <backslash><newline> will be "skipped".
229  // on the next <newline>, extra newlines will be added
230  std::ostringstream code;
231  unsigned int newlines = 0;
232  for (unsigned char ch = readChar(istr,bom); istr.good(); ch = readChar(istr,bom)) {
233  // Replace assorted special chars with spaces..
234  if (((ch & 0x80) == 0) && (ch != '\n') && (std::isspace(ch) || std::iscntrl(ch)))
235  ch = ' ';
236 
237  // <backslash><newline>..
238  // for gcc-compatibility the trailing spaces should be ignored
239  // for vs-compatibility the trailing spaces should be kept
240  // See tickets #640 and #1869
241  // The solution for now is to have a compiler-dependent behaviour.
242  if (ch == '\\') {
243  unsigned char chNext;
244 
245  std::string spaces;
246 
247 #ifdef __GNUC__
248  // gcc-compatibility: ignore spaces
249  for (;; spaces += ' ') {
250  chNext = (unsigned char)istr.peek();
251  if (chNext != '\n' && chNext != '\r' &&
252  (std::isspace(chNext) || std::iscntrl(chNext))) {
253  // Skip whitespace between <backslash> and <newline>
254  (void)readChar(istr,bom);
255  continue;
256  }
257 
258  break;
259  }
260 #else
261  // keep spaces
262  chNext = (unsigned char)istr.peek();
263 #endif
264  if (chNext == '\n' || chNext == '\r') {
265  ++newlines;
266  (void)readChar(istr,bom); // Skip the "<backslash><newline>"
267  } else {
268  code << "\\" << spaces;
269  }
270  } else {
271  code << char(ch);
272 
273  // if there has been <backslash><newline> sequences, add extra newlines..
274  if (ch == '\n' && newlines > 0) {
275  code << std::string(newlines, '\n');
276  newlines = 0;
277  }
278  }
279  }
280 
281  // ------------------------------------------------------------------------------------------
282  //
283  // Remove all comments..
284  std::string result = removeComments(code.str(), filename);
285  if (_settings.terminated())
286  return "";
287  code.str("");
288 
289  // ------------------------------------------------------------------------------------------
290  //
291  // Clean up all preprocessor statements
292  result = preprocessCleanupDirectives(result);
293  if (_settings.terminated())
294  return "";
295 
296  // ------------------------------------------------------------------------------------------
297  //
298  // Clean up preprocessor #if statements with Parentheses
299  result = removeParentheses(result);
300  if (_settings.terminated())
301  return "";
302 
303  // Remove '#if 0' blocks
304  if (result.find("#if 0\n") != std::string::npos)
305  result = removeIf0(result);
306  if (_settings.terminated())
307  return "";
308 
309  return result;
310 }
311 
312 
313 /** read preprocessor statements */
314 std::string Preprocessor::readpreprocessor(std::istream &istr, const unsigned int bom)
315 {
316  enum { NEWLINE, SPACE, PREPROCESSOR, BACKSLASH, OTHER } state = NEWLINE;
317  std::ostringstream code;
318  unsigned int newlines = 1;
319  unsigned char chPrev = ' ';
320  for (unsigned char ch = readChar(istr,bom); istr.good(); ch = readChar(istr,bom)) {
321  // Replace assorted special chars with spaces..
322  if (((ch & 0x80) == 0) && (ch != '\n') && (std::isspace(ch) || std::iscntrl(ch)))
323  ch = ' ';
324 
325  if (ch == ' ' && chPrev == ' ')
326  continue;
327  if (state == PREPROCESSOR && chPrev == '/' && (ch == '/' || ch == '*'))
328  state = OTHER;
329  chPrev = ch;
330 
331  if (ch == '\n') {
332  if (state != BACKSLASH) {
333  state = NEWLINE;
334  code << std::string(newlines, '\n');
335  newlines = 1;
336  } else {
337  ++newlines;
338  state = PREPROCESSOR;
339  }
340  continue;
341  }
342 
343  switch (state) {
344  case NEWLINE:
345  if (ch==' ')
346  state = SPACE;
347  else if (ch == '#') {
348  state = PREPROCESSOR;
349  code << ch;
350  } else
351  state = OTHER;
352  break;
353  case SPACE:
354  if (ch == '#') {
355  state = PREPROCESSOR;
356  code << ch;
357  } else if (ch != ' ')
358  state = OTHER;
359  break;
360  case PREPROCESSOR:
361  code << ch;
362  if (ch == '\\')
363  state = BACKSLASH;
364  break;
365  case BACKSLASH:
366  code << ch;
367  if (ch != ' ')
368  state = PREPROCESSOR;
369  break;
370  case OTHER:
371  break;
372  };
373  }
374 
375  std::string result = preprocessCleanupDirectives(code.str());
376  result = removeParentheses(result);
377  return removeIf0(result);
378 }
379 
380 std::string Preprocessor::preprocessCleanupDirectives(const std::string &processedFile)
381 {
382  std::ostringstream code;
383  std::istringstream sstr(processedFile);
384 
385  std::string line;
386  while (std::getline(sstr, line)) {
387  // Trim lines..
388  if (!line.empty() && line[0] == ' ')
389  line.erase(0, line.find_first_not_of(" "));
390  if (!line.empty() && line.back() == ' ')
391  line.erase(line.find_last_not_of(" ") + 1);
392 
393  // Preprocessor
394  if (!line.empty() && line[0] == '#') {
395  enum {
396  ESC_NONE,
397  ESC_SINGLE,
398  ESC_DOUBLE
399  } escapeStatus = ESC_NONE;
400 
401  char prev = ' '; // hack to make it skip spaces between # and the directive
402  code << "#";
403  std::string::const_iterator i = line.begin();
404  ++i;
405 
406  // need space.. #if( => #if (
407  bool needSpace = true;
408  while (i != line.end()) {
409  // disable esc-mode
410  if (escapeStatus != ESC_NONE) {
411  if (prev != '\\' && escapeStatus == ESC_SINGLE && *i == '\'') {
412  escapeStatus = ESC_NONE;
413  }
414  if (prev != '\\' && escapeStatus == ESC_DOUBLE && *i == '"') {
415  escapeStatus = ESC_NONE;
416  }
417  } else {
418  // enable esc-mode
419  if (escapeStatus == ESC_NONE && *i == '"')
420  escapeStatus = ESC_DOUBLE;
421  if (escapeStatus == ESC_NONE && *i == '\'')
422  escapeStatus = ESC_SINGLE;
423  }
424  // skip double whitespace between arguments
425  if (escapeStatus == ESC_NONE && prev == ' ' && *i == ' ') {
426  ++i;
427  continue;
428  }
429  // Convert #if( to "#if ("
430  if (escapeStatus == ESC_NONE) {
431  if (needSpace) {
432  if (*i == '(' || *i == '!')
433  code << " ";
434  else if (!std::isalpha((unsigned char)*i))
435  needSpace = false;
436  }
437  if (*i == '#')
438  needSpace = true;
439  }
440  code << *i;
441  if (escapeStatus != ESC_NONE && prev == '\\' && *i == '\\') {
442  prev = ' ';
443  } else {
444  prev = *i;
445  }
446  ++i;
447  }
448  if (escapeStatus != ESC_NONE) {
449  // unmatched quotes.. compiler should probably complain about this..
450  }
451  } else {
452  // Do not mess with regular code..
453  code << line;
454  }
455  code << (sstr.eof()?"":"\n");
456  }
457 
458  return code.str();
459 }
460 
461 static bool hasbom(const std::string &str)
462 {
463  return bool(str.size() >= 3 &&
464  static_cast<unsigned char>(str[0]) == 0xef &&
465  static_cast<unsigned char>(str[1]) == 0xbb &&
466  static_cast<unsigned char>(str[2]) == 0xbf);
467 }
468 
469 
470 // This wrapper exists because Sun's CC does not allow a static_cast
471 // from extern "C" int(*)(int) to int(*)(int).
472 static int tolowerWrapper(int c)
473 {
474  return std::tolower(c);
475 }
476 
477 
478 static bool isFallThroughComment(std::string comment)
479 {
480  // convert comment to lower case without whitespace
481  for (std::string::iterator i = comment.begin(); i != comment.end();) {
482  if (std::isspace(static_cast<unsigned char>(*i)))
483  i = comment.erase(i);
484  else
485  ++i;
486  }
487  std::transform(comment.begin(), comment.end(), comment.begin(), tolowerWrapper);
488 
489  return comment.find("fallthr") != std::string::npos ||
490  comment.find("fallsthr") != std::string::npos ||
491  comment.find("fall-thr") != std::string::npos ||
492  comment.find("dropthr") != std::string::npos ||
493  comment.find("passthr") != std::string::npos ||
494  comment.find("nobreak") != std::string::npos ||
495  comment == "fall";
496 }
497 
498 std::string Preprocessor::removeComments(const std::string &str, const std::string &filename)
499 {
500  // For the error report
501  unsigned int lineno = 1;
502 
503  // handling <backslash><newline>
504  // when this is encountered the <backslash><newline> will be "skipped".
505  // on the next <newline>, extra newlines will be added
506  unsigned int newlines = 0;
507  std::ostringstream code;
508  unsigned char previous = 0;
509  bool inPreprocessorLine = false;
510  std::vector<std::string> suppressionIDs;
511  const bool detectFallThroughComments = _settings.experimental && _settings.isEnabled("style");
512  bool fallThroughComment = false;
513 
514  for (std::string::size_type i = hasbom(str) ? 3U : 0U; i < str.length(); ++i) {
515  unsigned char ch = static_cast<unsigned char>(str[i]);
516  if (ch & 0x80) {
517  std::ostringstream errmsg;
518  errmsg << "(character code = 0x" << std::hex << (int(ch) & 0xff) << ")";
519  std::string info = errmsg.str();
520  errmsg.str("");
521  errmsg << "The code contains unhandled characters " << info << ". Checking continues, but do not expect valid results.\n"
522  << "The code contains characters that are unhandled " << info << ". Neither unicode nor extended ASCII are supported. Checking continues, but do not expect valid results.";
523  writeError(Path::simplifyPath(filename), lineno, _errorLogger, "unhandledCharacters", errmsg.str());
524  }
525 
526  if (_settings.terminated())
527  return "";
528 
529  // First skip over any whitespace that may be present
530  if (std::isspace(ch)) {
531  if (ch == ' ' && previous == ' ') {
532  // Skip double white space
533  } else {
534  code << char(ch);
535  previous = ch;
536  }
537 
538  // if there has been <backslash><newline> sequences, add extra newlines..
539  if (ch == '\n') {
540  if (previous != '\\')
541  inPreprocessorLine = false;
542  ++lineno;
543  if (newlines > 0) {
544  code << std::string(newlines, '\n');
545  newlines = 0;
546  previous = '\n';
547  }
548  }
549 
550  continue;
551  }
552 
553  if ((ch == '#') && (str.compare(i+1, 6, "error ") == 0 || str.compare(i+1, 8, "warning ") == 0)) {
554  if (str.compare(i, 6, "#error") == 0)
555  code << "#error";
556 
557  i = str.find('\n', i);
558  if (i == std::string::npos)
559  break;
560 
561  --i;
562  continue;
563  }
564 
565  // Remove comments..
566  if (str.compare(i, 2, "//") == 0) {
567  const std::size_t commentStart = i + 2;
568  i = str.find('\n', i);
569  if (i == std::string::npos)
570  break;
571  std::string comment(str, commentStart, i - commentStart);
572 
574  std::istringstream iss(comment);
575  std::string word;
576  iss >> word;
577  if (word == "cppcheck-suppress") {
578  iss >> word;
579  if (iss)
580  suppressionIDs.push_back(word);
581  }
582  }
583 
584  if (detectFallThroughComments && isFallThroughComment(comment)) {
585  fallThroughComment = true;
586  }
587 
588  code << "\n";
589  previous = '\n';
590  ++lineno;
591  } else if (str.compare(i, 2, "/*") == 0) {
592  const std::size_t commentStart = i + 2;
593  unsigned char chPrev = 0;
594  ++i;
595  while (i < str.length() && (chPrev != '*' || ch != '/')) {
596  chPrev = ch;
597  ++i;
598  ch = static_cast<unsigned char>(str[i]);
599  if (ch == '\n') {
600  ++newlines;
601  ++lineno;
602  }
603  }
604  std::string comment(str, commentStart, i - commentStart - 1);
605 
606  if (detectFallThroughComments && isFallThroughComment(comment)) {
607  fallThroughComment = true;
608  }
609 
611  std::istringstream iss(comment);
612  std::string word;
613  iss >> word;
614  if (word == "cppcheck-suppress") {
615  iss >> word;
616  if (iss)
617  suppressionIDs.push_back(word);
618  }
619  }
620  } else if ((i == 0 || std::isspace((unsigned char)str[i-1])) && str.compare(i, 5, "__asm") == 0) {
621  while (i < str.size() && (std::isalpha((unsigned char)str[i]) || str[i] == '_'))
622  code << str[i++];
623  while (i < str.size() && std::isspace((unsigned char)str[i])) {
624  if (str[i] == '\n')
625  lineno++;
626  code << str[i++];
627  }
628  if (str[i] == '{') {
629  // Ticket 4873: Extract comments from the __asm / __asm__'s content
630  std::string asmBody;
631  while (i < str.size() && str[i] != '}') {
632  if (str[i] == ';') {
633  std::string::size_type backslashN = str.find('\n', i);
634  if (backslashN != std::string::npos) // Ticket #4922: Don't go in infinite loop or crash if there is no '\n'
635  i = backslashN;
636  }
637  if (str[i] == '\n')
638  lineno++;
639  asmBody += str[i++];
640  }
641  code << removeComments(asmBody, filename);
642  code << '}';
643  } else
644  --i;
645  } else if (ch == '#' && previous == '\n') {
646  code << ch;
647  previous = ch;
648  inPreprocessorLine = true;
649 
650  // Add any pending inline suppressions that have accumulated.
651  if (!suppressionIDs.empty()) {
652  // Add the suppressions.
653  for (std::size_t j = 0; j < suppressionIDs.size(); ++j) {
654  const std::string errmsg(_settings.nomsg.addSuppression(suppressionIDs[j], filename, lineno));
655  if (!errmsg.empty()) {
656  writeError(filename, lineno, _errorLogger, "cppcheckError", errmsg);
657  }
658  }
659  suppressionIDs.clear();
660  }
661  } else {
662  if (!inPreprocessorLine) {
663  // Not whitespace, not a comment, and not preprocessor.
664  // Must be code here!
665 
666  // First check for a "fall through" comment match, but only
667  // add a suppression if the next token is 'case' or 'default'
668  if (detectFallThroughComments && fallThroughComment) {
669  const std::string::size_type j = str.find_first_not_of("abcdefghijklmnopqrstuvwxyz", i);
670  if (str.compare(i, j-i, "case") == 0 || str.compare(i, j-i, "default") == 0)
671  suppressionIDs.push_back("switchCaseFallThrough");
672  fallThroughComment = false;
673  }
674 
675  // Add any pending inline suppressions that have accumulated.
676  if (!suppressionIDs.empty()) {
677  // Relative filename
678  std::string relativeFilename(filename);
679  if (_settings.relativePaths) {
680  for (std::size_t j = 0U; j < _settings.basePaths.size(); ++j) {
681  const std::string bp = _settings.basePaths[j] + "/";
682  if (relativeFilename.compare(0,bp.size(),bp)==0) {
683  relativeFilename = relativeFilename.substr(bp.size());
684  }
685  }
686  }
687 
688  // Add the suppressions.
689  for (std::size_t j = 0; j < suppressionIDs.size(); ++j) {
690  const std::string errmsg(_settings.nomsg.addSuppression(suppressionIDs[j], relativeFilename, lineno));
691  if (!errmsg.empty()) {
692  writeError(filename, lineno, _errorLogger, "cppcheckError", errmsg);
693  }
694  }
695  suppressionIDs.clear();
696  }
697  }
698 
699  // C++14 digit separators
700  if (ch == '\'' && std::isxdigit(previous))
701  ; // Just skip it.
702 
703  // String or char constants..
704  else if (ch == '\"' || ch == '\'') {
705  code << char(ch);
706  char chNext;
707  do {
708  ++i;
709  chNext = str[i];
710  if (chNext == '\\') {
711  ++i;
712  const char chSeq = str[i];
713  if (chSeq == '\n')
714  ++newlines;
715  else {
716  code << chNext;
717  code << chSeq;
718  previous = static_cast<unsigned char>(chSeq);
719  }
720  } else {
721  code << chNext;
722  previous = static_cast<unsigned char>(chNext);
723  }
724  } while (i < str.length() && chNext != ch && chNext != '\n');
725  }
726 
727  // Rawstring..
728  else if (str.compare(i,2,"R\"")==0) {
729  std::string delim;
730  for (std::string::size_type i2 = i+2; i2 < str.length(); ++i2) {
731  if (i2 > 16 + i ||
732  std::isspace(str[i2]) ||
733  std::iscntrl(str[i2]) ||
734  str[i2] == ')' ||
735  str[i2] == '\\') {
736  delim = " ";
737  break;
738  } else if (str[i2] == '(')
739  break;
740 
741  delim += str[i2];
742  }
743  const std::string::size_type endpos = str.find(")" + delim + "\"", i);
744  if (delim != " " && endpos != std::string::npos) {
745  unsigned int rawstringnewlines = 0;
746  code << '\"';
747  for (std::string::size_type p = i + 3 + delim.size(); p < endpos; ++p) {
748  if (str[p] == '\n') {
749  rawstringnewlines++;
750  code << "\\n";
751  } else if (std::iscntrl((unsigned char)str[p]) ||
752  std::isspace((unsigned char)str[p])) {
753  code << " ";
754  } else if (str[p] == '\\') {
755  code << "\\\\";
756  } else if (str[p] == '\"') {
757  code << "\\" << (char)str[p];
758  } else {
759  code << (char)str[p];
760  }
761  }
762  code << "\"";
763  if (rawstringnewlines > 0)
764  code << std::string(rawstringnewlines, '\n');
765  i = endpos + delim.size() + 1;
766  } else {
767  code << "R";
768  previous = 'R';
769  }
770  } else {
771  code << char(ch);
772  previous = ch;
773  }
774  }
775  }
776 
777  return code.str();
778 }
779 
780 std::string Preprocessor::removeIf0(const std::string &code)
781 {
782  std::ostringstream ret;
783  std::istringstream istr(code);
784  std::string line;
785  while (std::getline(istr,line)) {
786  ret << line << "\n";
787  if (line == "#if 0") {
788  // goto the end of the '#if 0' block
789  unsigned int level = 1;
790  bool in = false;
791  while (level > 0 && std::getline(istr,line)) {
792  if (line.compare(0,3,"#if") == 0)
793  ++level;
794  else if (line == "#endif")
795  --level;
796  else if ((line == "#else") || (line.compare(0, 5, "#elif") == 0)) {
797  if (level == 1)
798  in = true;
799  } else {
800  if (in)
801  ret << line << "\n";
802  else
803  // replace code within '#if 0' block with empty lines
804  ret << "\n";
805  continue;
806  }
807 
808  ret << line << "\n";
809  }
810  }
811  }
812  return ret.str();
813 }
814 
815 
816 std::string Preprocessor::removeParentheses(const std::string &str)
817 {
818  if (str.find("\n#if") == std::string::npos && str.compare(0, 3, "#if") != 0)
819  return str;
820 
821  std::istringstream istr(str);
822  std::ostringstream ret;
823  std::string line;
824  while (std::getline(istr, line)) {
825  if (line.compare(0, 3, "#if") == 0 || line.compare(0, 5, "#elif") == 0) {
826  std::string::size_type pos;
827  pos = 0;
828  while ((pos = line.find(" (", pos)) != std::string::npos)
829  line.erase(pos, 1);
830  pos = 0;
831  while ((pos = line.find("( ", pos)) != std::string::npos)
832  line.erase(pos + 1, 1);
833  pos = 0;
834  while ((pos = line.find(" )", pos)) != std::string::npos)
835  line.erase(pos, 1);
836  pos = 0;
837  while ((pos = line.find(") ", pos)) != std::string::npos)
838  line.erase(pos + 1, 1);
839 
840  // Remove inner parentheses "((..))"..
841  pos = 0;
842  while ((pos = line.find("((", pos)) != std::string::npos) {
843  ++pos;
844  std::string::size_type pos2 = line.find_first_of("()", pos + 1);
845  if (pos2 != std::string::npos && line[pos2] == ')') {
846  line.erase(pos2, 1);
847  line.erase(pos, 1);
848  }
849  }
850 
851  // "#if(A) => #if A", but avoid "#if (defined A) || defined (B)"
852  if ((line.compare(0, 4, "#if(") == 0 || line.compare(0, 6, "#elif(") == 0) &&
853  line[line.length() - 1] == ')') {
854  int ind = 0;
855  for (std::string::size_type i = 0; i < line.length(); ++i) {
856  if (line[i] == '(')
857  ++ind;
858  else if (line[i] == ')') {
859  --ind;
860  if (ind == 0) {
861  if (i == line.length() - 1) {
862  line[line.find('(')] = ' ';
863  line.erase(line.length() - 1);
864  }
865  break;
866  }
867  }
868  }
869  }
870 
871  if (line.compare(0, 4, "#if(") == 0)
872  line.insert(3, " ");
873  else if (line.compare(0, 6, "#elif(") == 0)
874  line.insert(5, " ");
875  }
876  ret << line << "\n";
877  }
878 
879  return ret.str();
880 }
881 
882 
883 void Preprocessor::removeAsm(std::string &str)
884 {
885  std::string::size_type pos = 0;
886  while ((pos = str.find("#asm\n", pos)) != std::string::npos) {
887  str.replace(pos, 4, "asm(");
888 
889  std::string::size_type pos2 = str.find("#endasm", pos);
890  if (pos2 != std::string::npos) {
891  str.replace(pos2, 7, ");");
892  pos = pos2;
893  }
894  }
895 }
896 
897 
898 void Preprocessor::preprocess(std::istream &istr, std::map<std::string, std::string> &result, const std::string &filename, const std::list<std::string> &includePaths)
899 {
900  std::list<std::string> configs;
901  std::string data;
902  preprocess(istr, data, configs, filename, includePaths);
903  for (std::list<std::string>::const_iterator it = configs.begin(); it != configs.end(); ++it) {
904  if (_settings.userUndefs.find(*it) == _settings.userUndefs.end()) {
905  result[ *it ] = getcode(data, *it, filename);
906  }
907  }
908 }
909 
910 std::string Preprocessor::removeSpaceNearNL(const std::string &str)
911 {
912  std::string tmp;
913  char prev = '\n'; // treat start of file as newline
914  for (std::size_t i = 0; i < str.size(); i++) {
915  if (str[i] == ' ' &&
916  (prev == '\n' ||
917  i + 1 >= str.size() || // treat end of file as newline
918  str[i+1] == '\n'
919  )
920  ) {
921  // Ignore space that has new line in either side of it
922  } else {
923  tmp.append(1, str[i]);
924  prev = str[i];
925  }
926  }
927 
928  return tmp;
929 }
930 
931 void Preprocessor::replaceIfDefined(std::string &str) const
932 {
933  std::string::size_type pos = 0;
934  while ((pos = str.find("#if defined(", pos)) != std::string::npos) {
935  std::string::size_type pos2 = str.find(')', pos + 9);
936  if (pos2 > str.length() - 1)
937  break;
938  if (str[pos2+1] == '\n') {
939  str.erase(pos2, 1);
940  str.erase(pos + 3, 9);
941  str.insert(pos + 3, "def ");
942  }
943  ++pos;
944 
945  if (_settings.terminated())
946  return;
947  }
948 
949  pos = 0;
950  while ((pos = str.find("#if !defined(", pos)) != std::string::npos) {
951  std::string::size_type pos2 = str.find(')', pos + 9);
952  if (pos2 > str.length() - 1)
953  break;
954  if (str[pos2+1] == '\n') {
955  str.erase(pos2, 1);
956  str.erase(pos + 3, 10);
957  str.insert(pos + 3, "ndef ");
958  }
959  ++pos;
960 
961  if (_settings.terminated())
962  return;
963  }
964 
965  pos = 0;
966  while ((pos = str.find("#elif defined(", pos)) != std::string::npos) {
967  std::string::size_type pos2 = str.find(')', pos + 9);
968  if (pos2 > str.length() - 1)
969  break;
970  if (str[pos2+1] == '\n') {
971  str.erase(pos2, 1);
972  str.erase(pos + 6, 8);
973  }
974  ++pos;
975 
976  if (_settings.terminated())
977  return;
978  }
979 }
980 
981 void Preprocessor::preprocessWhitespaces(std::string &processedFile)
982 {
983  // Replace all tabs with spaces..
984  std::replace(processedFile.begin(), processedFile.end(), '\t', ' ');
985 
986  // Remove space characters that are after or before new line character
987  processedFile = removeSpaceNearNL(processedFile);
988 }
989 
990 void Preprocessor::preprocess(std::istream &srcCodeStream, std::string &processedFile, std::list<std::string> &resultConfigurations, const std::string &filename, const std::list<std::string> &includePaths)
991 {
992  std::string forcedIncludes;
993 
994  if (file0.empty())
995  file0 = filename;
996 
997  processedFile = read(srcCodeStream, filename);
998 
999  for (std::list<std::string>::iterator it = _settings.userIncludes.begin();
1000  it != _settings.userIncludes.end();
1001  ++it) {
1002  const std::string& cur = *it;
1003 
1004  // try to open file
1005  std::ifstream fin;
1006 
1007  fin.open(cur.c_str());
1008  if (!fin.is_open()) {
1009  missingInclude(cur,
1010  1,
1011  cur,
1012  UserHeader
1013  );
1014  continue;
1015  }
1016  const std::string fileData = read(fin, filename);
1017 
1018  fin.close();
1019 
1020  forcedIncludes +=
1021  "#file \"" + cur + "\"\n" +
1022  "#line 1\n" +
1023  fileData + "\n" +
1024  "#endfile\n"
1025  ;
1026  }
1027 
1028  for (std::vector<std::string>::iterator it = _settings.library.defines.begin();
1029  it != _settings.library.defines.end();
1030  ++it) {
1031  forcedIncludes += *it;
1032  }
1033 
1034  if (!forcedIncludes.empty()) {
1035  processedFile =
1036  forcedIncludes +
1037  "#file \"" + filename + "\"\n" +
1038  "#line 1\n" +
1039  processedFile +
1040  "#endfile\n"
1041  ;
1042  }
1043 
1044  // Remove asm(...)
1045  removeAsm(processedFile);
1046 
1047  // Replace "defined A" with "defined(A)"
1048  {
1049  std::istringstream istr(processedFile);
1050  std::ostringstream ostr;
1051  std::string line;
1052  while (std::getline(istr, line)) {
1053  if (line.compare(0, 4, "#if ") == 0 || line.compare(0, 6, "#elif ") == 0) {
1054  std::string::size_type pos = 0;
1055  while ((pos = line.find(" defined ")) != std::string::npos) {
1056  line[pos+8] = '(';
1057  pos = line.find_first_of(" |&", pos + 8);
1058  if (pos == std::string::npos)
1059  line += ")";
1060  else
1061  line.insert(pos, ")");
1062 
1063  if (_settings.terminated())
1064  return;
1065  }
1066  }
1067  ostr << line << "\n";
1068  }
1069  processedFile = ostr.str();
1070  }
1071 
1072  std::map<std::string, std::string> defs(getcfgmap(_settings.userDefines, &_settings, filename));
1073 
1074  if (_settings.maxConfigs == 1U) {
1075  std::set<std::string> pragmaOnce;
1076  std::list<std::string> includes;
1077  processedFile = handleIncludes(processedFile, filename, includePaths, defs, pragmaOnce, includes);
1078  resultConfigurations = getcfgs(processedFile, filename, defs);
1079  } else {
1080  handleIncludes(processedFile, filename, includePaths);
1081 
1082  replaceIfDefined(processedFile);
1083 
1084  // Get all possible configurations..
1085  resultConfigurations = getcfgs(processedFile, filename, defs);
1086 
1087  // Remove configurations that are disabled by -U
1088  handleUndef(resultConfigurations);
1089  }
1090 }
1091 
1092 void Preprocessor::handleUndef(std::list<std::string> &configurations) const
1093 {
1094  if (!_settings.userUndefs.empty()) {
1095  for (std::list<std::string>::iterator cfg = configurations.begin(); cfg != configurations.end();) {
1096  bool undef = false;
1097  for (std::set<std::string>::const_iterator it = _settings.userUndefs.begin(); it != _settings.userUndefs.end(); ++it) {
1098  if (*it == *cfg)
1099  undef = true;
1100  else if (cfg->compare(0,it->length(),*it)==0 && cfg->find_first_of(";=") == it->length())
1101  undef = true;
1102  else if (cfg->find(";" + *it) == std::string::npos)
1103  continue;
1104  else if (cfg->find(";" + *it + ";") != std::string::npos)
1105  undef = true;
1106  else if (cfg->find(";" + *it + "=") != std::string::npos)
1107  undef = true;
1108  else if (cfg->find(";" + *it) + it->size() + 1U == cfg->size())
1109  undef = true;
1110  if (undef)
1111  break;
1112  }
1113 
1114  if (undef)
1115  configurations.erase(cfg++);
1116  else
1117  ++cfg;
1118  }
1119  }
1120 }
1121 
1122 // Get the DEF in this line: "#ifdef DEF"
1123 std::string Preprocessor::getdef(std::string line, bool def)
1124 {
1125  if (line.empty() || line[0] != '#')
1126  return "";
1127 
1128  // If def is true, the line must start with "#ifdef"
1129  if (def && line.compare(1, 6, "ifdef ") != 0 && line.compare(1, 3, "if ") != 0
1130  && (line.compare(1, 5, "elif ") != 0 || line.compare(1, 6, "elif !") == 0)) {
1131  return "";
1132  }
1133 
1134  // If def is false, the line must start with "#ifndef"
1135  if (!def && line.compare(1, 7, "ifndef ") != 0 && line.compare(1, 6, "elif !") != 0) {
1136  return "";
1137  }
1138 
1139  // Remove the "#ifdef" or "#ifndef"
1140  if (line.compare(1, 11, "if defined ") == 0)
1141  line.erase(0, 11);
1142  else if (line.compare(1, 14, "elif !defined(") == 0) {
1143  line.erase(0, 15);
1144  std::string::size_type pos = line.find(')');
1145  // if pos == ::npos then another part of the code will complain
1146  // about the mismatch
1147  if (pos != std::string::npos)
1148  line.erase(pos, 1);
1149  } else
1150  line.erase(0, line.find(' '));
1151 
1152  // Remove all spaces.
1153  std::string::size_type pos = 0;
1154  while ((pos = line.find(' ', pos)) != std::string::npos) {
1155  const unsigned char chprev(static_cast<unsigned char>((pos > 0) ? line[pos-1] : 0));
1156  const unsigned char chnext(static_cast<unsigned char>((pos + 1 < line.length()) ? line[pos+1] : 0));
1157  if ((std::isalnum(chprev) || chprev == '_') && (std::isalnum(chnext) || chnext == '_'))
1158  ++pos;
1159  else
1160  line.erase(pos, 1);
1161  }
1162 
1163  // The remaining string is our result.
1164  return line;
1165 }
1166 
1167 /** Simplify variable in variable map. */
1168 static Token *simplifyVarMapExpandValue(Token *tok, const std::map<std::string, std::string> &variables, std::set<std::string> seenVariables, const Settings& settings)
1169 {
1170  // TODO: handle function-macros too.
1171 
1172  // Prevent infinite recursion..
1173  if (seenVariables.find(tok->str()) != seenVariables.end())
1174  return tok;
1175  seenVariables.insert(tok->str());
1176 
1177  const std::map<std::string, std::string>::const_iterator it = variables.find(tok->str());
1178  if (it != variables.end()) {
1179  TokenList tokenList(&settings);
1180  std::istringstream istr(it->second);
1181  if (tokenList.createTokens(istr)) {
1182  // expand token list
1183  for (Token *tok2 = tokenList.front(); tok2; tok2 = tok2->next()) {
1184  if (tok2->isName()) {
1185  tok2 = simplifyVarMapExpandValue(tok2, variables, seenVariables, settings);
1186  }
1187  }
1188 
1189  // insert token list into "parent" token list
1190  for (const Token *tok2 = tokenList.front(); tok2; tok2 = tok2->next()) {
1191  if (tok2->previous()) {
1192  tok->insertToken(tok2->str());
1193  tok = tok->next();
1194  } else
1195  tok->str(tok2->str());
1196  }
1197  }
1198  }
1199 
1200  return tok;
1201 }
1202 
1203 /**
1204  * Simplifies the variable map. For example if the map contains A=>B, B=>1, then A=>B is simplified to A=>1.
1205  * @param [in,out] variables - a map of variable name to variable value. This map will be modified.
1206  * @param [in] settings Current settings being used
1207  */
1208 static void simplifyVarMap(std::map<std::string, std::string> &variables, const Settings& settings)
1209 {
1210  for (std::map<std::string, std::string>::iterator i = variables.begin(); i != variables.end(); ++i) {
1211  TokenList tokenList(&settings);
1212  std::istringstream istr(i->second);
1213  if (tokenList.createTokens(istr)) {
1214  for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
1215  if (tok->isName()) {
1216  std::set<std::string> seenVariables;
1217  tok = simplifyVarMapExpandValue(tok, variables, seenVariables, settings);
1218  }
1219  }
1220 
1221  std::string str;
1222  for (const Token *tok = tokenList.front(); tok; tok = tok->next())
1223  str.append((tok->previous() ? " " : "") + tok->str());
1224  i->second = str;
1225  }
1226  }
1227 }
1228 
1229 std::list<std::string> Preprocessor::getcfgs(const std::string &filedata, const std::string &filename, const std::map<std::string, std::string> &defs)
1230 {
1231  std::list<std::string> ret;
1232  ret.push_back("");
1233 
1234  std::list<std::string> deflist, ndeflist;
1235 
1236  // constants defined through "#define" in the code..
1237  std::set<std::string> defines;
1238  std::map<std::string, std::string> alldefinesmap(defs);
1239  std::stack<std::pair<std::string,bool> > includeStack;
1240  includeStack.push(std::pair<std::string,bool>(filename,false));
1241 
1242  // How deep into included files are we currently parsing?
1243  // 0=>Source file, 1=>Included by source file, 2=>included by header that was included by source file, etc
1244  int filelevel = 0;
1245 
1246  bool includeguard = false;
1247  unsigned int linenr = 0;
1248  std::istringstream istr(filedata);
1249  std::string line;
1250  const bool printDebug = _settings.debugwarnings;
1251  while (std::getline(istr, line)) {
1252  ++linenr;
1253 
1254  if (_settings.terminated())
1255  return ret;
1256 
1257  if (_errorLogger)
1258  _errorLogger->reportProgress(filename, "Preprocessing (get configurations 1)", 0);
1259 
1260  if (line.empty())
1261  continue;
1262 
1263  if (line.compare(0, 6, "#file ") == 0) {
1264  includeguard = true;
1265  const std::string::size_type start=line.find('\"');
1266  const std::string::size_type end=line.find('\"',start+1);
1267  const std::string includeFile=line.substr(start+1,end-start-1);
1268  ++filelevel;
1269  bool fileExcluded = _settings.configurationExcluded(includeFile);
1270  includeStack.push(std::pair<std::string,bool>(includeFile,fileExcluded));
1271  continue;
1272  }
1273 
1274  else if (line == "#endfile") {
1275  includeguard = false;
1276  includeStack.pop();
1277  if (filelevel > 0)
1278  --filelevel;
1279  continue;
1280  }
1281 
1282  if (line.compare(0, 8, "#define ") == 0) {
1283  bool valid = false;
1284  for (std::string::size_type pos = 8; pos < line.size(); ++pos) {
1285  const char ch = line[pos];
1286  if (ch=='_' || (ch>='a' && ch<='z') || (ch>='A' && ch<='Z') || (pos>8 && ch>='0' && ch<='9')) {
1287  valid = true;
1288  continue;
1289  }
1290  if (ch==' ' || ch=='(') {
1291  if (valid)
1292  break;
1293  }
1294  valid = false;
1295  break;
1296  }
1297  if (!valid)
1298  line.clear();
1299  else {
1300  std::string definestr = line.substr(8);
1301  const std::string::size_type spacepos = definestr.find(' ');
1302  if (spacepos != std::string::npos)
1303  definestr[spacepos] = '=';
1304  defines.insert(definestr);
1305 
1306  const std::string::size_type separatorpos = definestr.find_first_of("=(");
1307  if (separatorpos != std::string::npos && definestr[separatorpos] == '=') {
1308  const std::string varname(definestr.substr(0, separatorpos));
1309  const std::string value(definestr.substr(separatorpos + 1));
1310  alldefinesmap[varname] = value;
1311  }
1312  }
1313  }
1314 
1315  if (!line.empty() && line.compare(0, 3, "#if") != 0)
1316  includeguard = false;
1317 
1318  if (line.empty() || line[0] != '#')
1319  continue;
1320 
1321  if (includeguard)
1322  continue;
1323 
1324  if (line.compare(0, 5, "#line") == 0)
1325  continue;
1326 
1327  bool from_negation = false;
1328 
1329  std::string def = getdef(line, true);
1330  if (def.empty()) {
1331  def = getdef(line, false);
1332  // sub conditionals of ndef blocks need to be
1333  // constructed _without_ the negated define
1334  if (!def.empty())
1335  from_negation = true;
1336  }
1337  if (!def.empty()) {
1338  int par = 0;
1339  for (std::string::size_type pos = 0; pos < def.length(); ++pos) {
1340  if (def[pos] == '(')
1341  ++par;
1342  else if (def[pos] == ')') {
1343  --par;
1344  if (par < 0)
1345  break;
1346  }
1347  }
1348  if (par != 0) {
1349  std::ostringstream lineStream;
1350  lineStream << __LINE__;
1351  const std::string errorId = "preprocessor" + lineStream.str();
1352  const std::string errorText = "mismatching number of '(' and ')' in this line: " + def;
1353  writeError(filename, linenr, _errorLogger, errorId, errorText);
1354  ret.clear();
1355  return ret;
1356  }
1357 
1358  // Replace defined constants
1359  simplifyCondition(alldefinesmap, def, false);
1360 
1361  if (! deflist.empty() && line.compare(0, 6, "#elif ") == 0)
1362  deflist.pop_back();
1363 
1364  // translate A==1 condition to A=1 configuration
1365  if (def.find("==") != std::string::npos) {
1366  // Check if condition match pattern "%name% == %num%"
1367  // %name%
1368  std::string::size_type pos = 0;
1369  if (std::isalpha((unsigned char)def[pos]) || def[pos] == '_') {
1370  ++pos;
1371  while (std::isalnum((unsigned char)def[pos]) || def[pos] == '_')
1372  ++pos;
1373  }
1374 
1375  // ==
1376  if (def.compare(pos,2,"==")==0)
1377  pos += 2;
1378 
1379  // %num%
1380  if (pos<def.size() && std::isdigit(def[pos])) {
1381  if (def.compare(pos,2,"0x")==0) {
1382  pos += 2;
1383  if (pos >= def.size())
1384  pos = 0;
1385  while (pos < def.size() && std::isxdigit((unsigned char)def[pos]))
1386  ++pos;
1387  } else {
1388  while (pos < def.size() && std::isdigit((unsigned char)def[pos]))
1389  ++pos;
1390  }
1391 
1392  // Does the condition match the pattern "%name% == %num%"?
1393  if (pos == def.size()) {
1394  def.erase(def.find("=="),1);
1395  }
1396  }
1397  }
1398 
1399  deflist.push_back(def);
1400  def = "";
1401 
1402  for (std::list<std::string>::const_iterator it = deflist.begin(); it != deflist.end(); ++it) {
1403  if (*it == "0")
1404  break;
1405  if (*it == "1" || *it == "!")
1406  continue;
1407 
1408  // don't add "T;T":
1409  // treat two and more similar nested conditions as one
1410  if (def != *it) {
1411  if (! def.empty())
1412  def += ";";
1413  def += *it;
1414  }
1415 
1416  /* TODO: Fix TestPreprocessor::test7e (#2552)
1417  else
1418  {
1419  std::ostringstream lineStream;
1420  lineStream << __LINE__;
1421 
1422  ErrorLogger::ErrorMessage errmsg;
1423  ErrorLogger::ErrorMessage::FileLocation loc;
1424  loc.setfile(filename);
1425  loc.line = linenr;
1426  errmsg._callStack.push_back(loc);
1427  errmsg._severity = Severity::error;
1428  errmsg.setmsg(*it+" is already guaranteed to be defined");
1429  errmsg._id = "preprocessor" + lineStream.str();
1430  _errorLogger->reportErr(errmsg);
1431  }
1432  */
1433  }
1434  if (from_negation) {
1435  ndeflist.push_back(deflist.back());
1436  deflist.back() = "!";
1437  }
1438 
1439  if (std::find(ret.begin(), ret.end(), def) == ret.end()) {
1440  if (!includeStack.top().second) {
1441  ret.push_back(def);
1442  } else {
1443  if (_errorLogger && printDebug) {
1444  std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
1445  const ErrorLogger::ErrorMessage errmsg(locationList, Severity::debug,
1446  "Configuration not considered: " + def +" for file:"+includeStack.top().first, "debug", false);
1447  _errorLogger->reportErr(errmsg);
1448  }
1449  }
1450  }
1451  }
1452 
1453  else if (line.compare(0, 5, "#else") == 0 && ! deflist.empty()) {
1454  if (deflist.back() == "!" && !ndeflist.empty()) {
1455  deflist.back() = ndeflist.back();
1456  ndeflist.pop_back();
1457  } else {
1458  std::string tempDef((deflist.back() == "1") ? "0" : "1");
1459  deflist.back() = tempDef;
1460  }
1461  }
1462 
1463  else if (line.compare(0, 6, "#endif") == 0 && ! deflist.empty()) {
1464  if (deflist.back() == "!" && !ndeflist.empty())
1465  ndeflist.pop_back();
1466  deflist.pop_back();
1467  }
1468  }
1469 
1470  // Remove defined constants from ifdef configurations..
1471  std::size_t count = 0;
1472  for (std::list<std::string>::iterator it = ret.begin(); it != ret.end(); ++it) {
1473  if (_errorLogger)
1474  _errorLogger->reportProgress(filename, "Preprocessing (get configurations 2)", (100 * count++) / ret.size());
1475 
1476  std::string cfg(*it);
1477  for (std::set<std::string>::const_iterator it2 = defines.begin(); it2 != defines.end(); ++it2) {
1478  std::string::size_type pos = 0;
1479 
1480  // Get name of define
1481  std::string defineName(*it2);
1482  const std::string::size_type end = defineName.find_first_of("=(");
1483  if (end != std::string::npos)
1484  defineName.erase(end);
1485 
1486  // Remove ifdef configurations that match the defineName
1487  while ((pos = cfg.find(defineName, pos)) != std::string::npos) {
1488  const std::string::size_type pos1 = pos;
1489  ++pos;
1490  if (pos1 > 0 && cfg[pos1-1] != ';')
1491  continue;
1492  const std::string::size_type pos2 = pos1 + defineName.length();
1493  if (pos2 < cfg.length() && cfg[pos2] != ';')
1494  continue;
1495  --pos;
1496  cfg.erase(pos, defineName.length());
1497  }
1498  }
1499  if (cfg.length() != it->length()) {
1500  while (cfg.length() > 0 && cfg[0] == ';')
1501  cfg.erase(0, 1);
1502 
1503  while (cfg.length() > 0 && cfg.back() == ';')
1504  cfg.erase(cfg.length() - 1);
1505 
1506  std::string::size_type pos = 0;
1507  while ((pos = cfg.find(";;", pos)) != std::string::npos)
1508  cfg.erase(pos, 1);
1509 
1510  *it = cfg;
1511  }
1512  }
1513 
1514  // convert configurations: "defined(A) && defined(B)" => "A;B"
1515  for (std::list<std::string>::iterator it = ret.begin(); it != ret.end(); ++it) {
1516  std::string s(*it);
1517 
1518  if (s.find("&&") != std::string::npos) {
1519  Tokenizer tokenizer(&_settings, _errorLogger);
1520  if (!tokenizer.tokenizeCondition(s)) {
1521  std::ostringstream lineStream;
1522  lineStream << __LINE__;
1523 
1526  loc.setfile(filename);
1527  loc.line = 1;
1528  errmsg._callStack.push_back(loc);
1529  errmsg._severity = Severity::error;
1530  errmsg.setmsg("Error parsing this: " + s);
1531  errmsg._id = "preprocessor" + lineStream.str();
1532  _errorLogger->reportErr(errmsg);
1533  }
1534 
1535 
1536  const Token *tok = tokenizer.tokens();
1537  std::set<std::string> varList;
1538  while (tok) {
1539  if (Token::Match(tok, "defined ( %name% )")) {
1540  varList.insert(tok->strAt(2));
1541  tok = tok->tokAt(4);
1542  if (tok && tok->str() == "&&") {
1543  tok = tok->next();
1544  }
1545  } else if (Token::Match(tok, "%name% ;")) {
1546  varList.insert(tok->str());
1547  tok = tok->tokAt(2);
1548  } else {
1549  break;
1550  }
1551  }
1552 
1553  s = join(varList, ';');
1554 
1555  if (!s.empty())
1556  *it = s;
1557  }
1558  }
1559 
1560  // Convert configurations into a canonical form: B;C;A or C;A;B => A;B;C
1561  for (std::list<std::string>::iterator it = ret.begin(); it != ret.end(); ++it)
1562  *it = unify(*it, ';');
1563 
1564  // Remove duplicates from the ret list..
1565  ret.sort();
1566  ret.unique();
1567 
1568  // C code => remove __cplusplus configurations..
1569  if (!cplusplus(&_settings, filename) && Path::isC(filename)) {
1570  for (std::list<std::string>::iterator it = ret.begin(); it != ret.end();) {
1571  if (it->find("__cplusplus") != std::string::npos) {
1572  ret.erase(it++);
1573  } else {
1574  ++it;
1575  }
1576  }
1577  }
1578 
1579  // cleanup unhandled configurations..
1580  for (std::list<std::string>::iterator it = ret.begin(); it != ret.end();) {
1581  const std::string s(*it + ";");
1582 
1583  bool unhandled = false;
1584 
1585  for (std::string::size_type pos = 0; pos < s.length(); ++pos) {
1586  const unsigned char c = static_cast<unsigned char>(s[pos]);
1587 
1588  // ok with ";"
1589  if (c == ';')
1590  continue;
1591 
1592  // identifier..
1593  if (std::isalpha(c) || c == '_') {
1594  while (std::isalnum((unsigned char)s[pos]) || s[pos] == '_')
1595  ++pos;
1596  if (s[pos] == '=') {
1597  ++pos;
1598  while (std::isdigit((unsigned char)s[pos]))
1599  ++pos;
1600  if (s[pos] != ';') {
1601  unhandled = true;
1602  break;
1603  }
1604  }
1605 
1606  --pos;
1607  continue;
1608  }
1609 
1610  // not ok..
1611  else {
1612  unhandled = true;
1613  break;
1614  }
1615  }
1616 
1617  if (unhandled) {
1618  // unhandled ifdef configuration..
1619  if (_errorLogger && printDebug) {
1620  std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
1621  const ErrorLogger::ErrorMessage errmsg(locationList, Severity::debug, "unhandled configuration: " + *it, "debug", false);
1622  _errorLogger->reportErr(errmsg);
1623  }
1624 
1625  ret.erase(it++);
1626  } else {
1627  ++it;
1628  }
1629  }
1630 
1631  return ret;
1632 }
1633 
1634 
1635 void Preprocessor::simplifyCondition(const std::map<std::string, std::string> &cfg, std::string &condition, bool match)
1636 {
1637  Tokenizer tokenizer(&_settings, _errorLogger);
1638  if (!tokenizer.tokenizeCondition("(" + condition + ")")) {
1639  // If tokenize returns false, then there is syntax error in the
1640  // code which we can't handle. So stop here.
1641  return;
1642  }
1643 
1644  if (Token::Match(tokenizer.tokens(), "( %name% )")) {
1645  std::map<std::string,std::string>::const_iterator var = cfg.find(tokenizer.tokens()->strAt(1));
1646  if (var != cfg.end()) {
1647  const std::string &value = var->second;
1648  condition = (value == "0") ? "0" : "1";
1649  } else if (match)
1650  condition = "0";
1651  return;
1652  }
1653 
1654  if (Token::Match(tokenizer.tokens(), "( ! %name% )")) {
1655  std::map<std::string,std::string>::const_iterator var = cfg.find(tokenizer.tokens()->strAt(2));
1656 
1657  if (var == cfg.end())
1658  condition = "1";
1659  else if (var->second == "0")
1660  condition = "1";
1661  else if (match)
1662  condition = "0";
1663  return;
1664  }
1665 
1666  // replace variable names with values..
1667  for (Token *tok = const_cast<Token *>(tokenizer.tokens()); tok; tok = tok->next()) {
1668  if (!tok->isName())
1669  continue;
1670 
1671  if (Token::Match(tok, "defined ( %name% )")) {
1672  if (cfg.find(tok->strAt(2)) != cfg.end())
1673  tok->str("1");
1674  else if (match)
1675  tok->str("0");
1676  else
1677  continue;
1678  tok->deleteNext(3);
1679  continue;
1680  }
1681 
1682  if (Token::Match(tok, "defined %name%")) {
1683  if (cfg.find(tok->strAt(1)) != cfg.end())
1684  tok->str("1");
1685  else if (match)
1686  tok->str("0");
1687  else
1688  continue;
1689  tok->deleteNext();
1690  continue;
1691  }
1692 
1693  const std::map<std::string, std::string>::const_iterator it = cfg.find(tok->str());
1694  if (it != cfg.end()) {
1695  if (!it->second.empty()) {
1696  // Tokenize the value
1697  Tokenizer tokenizer2(&_settings, _errorLogger);
1698  tokenizer2.tokenizeCondition(it->second);
1699 
1700  // Copy the value tokens
1701  std::stack<Token *> link;
1702  for (const Token *tok2 = tokenizer2.tokens(); tok2; tok2 = tok2->next()) {
1703  tok->str(tok2->str());
1704 
1705  if (Token::Match(tok2,"[{([]"))
1706  link.push(tok);
1707  else if (!link.empty() && Token::Match(tok2,"[})]]")) {
1708  Token::createMutualLinks(link.top(), tok);
1709  link.pop();
1710  }
1711 
1712  if (tok2->next()) {
1713  tok->insertToken("");
1714  tok = tok->next();
1715  }
1716  }
1717  } else if ((!tok->previous() || Token::Match(tok->previous(), "&&|%oror%|(")) &&
1718  (!tok->next() || Token::Match(tok->next(), "&&|%oror%|)")))
1719  tok->str("1");
1720  else
1721  tok->deleteThis();
1722  }
1723  }
1724 
1725  // simplify calculations..
1727  bool modified = true;
1728  while (modified) {
1729  modified = false;
1730  modified |= tokenizer.simplifySizeof();
1731  modified |= tokenizer.simplifyCalculations();
1732  modified |= tokenizer.simplifyConstTernaryOp();
1733  modified |= tokenizer.simplifyRedundantParentheses();
1734  for (Token *tok = const_cast<Token *>(tokenizer.tokens()); tok; tok = tok->next()) {
1735  if (Token::Match(tok, "! %num%")) {
1736  tok->deleteThis();
1737  tok->str(tok->str() == "0" ? "1" : "0");
1738  modified = true;
1739  }
1740  }
1741  }
1742 
1743  for (Token *tok = const_cast<Token *>(tokenizer.tokens()); tok; tok = tok->next()) {
1744  if (Token::Match(tok, "(|%oror%|&& %num% &&|%oror%|)")) {
1745  if (tok->next()->str() != "0") {
1746  tok->next()->str("1");
1747  }
1748  }
1749  }
1750 
1751  for (Token *tok = const_cast<Token *>(tokenizer.tokens()); tok; tok = tok->next()) {
1752  while (Token::Match(tok, "(|%oror% %any% %oror% 1")) {
1753  tok->deleteNext(2);
1754  if (tok->tokAt(-3))
1755  tok = tok->tokAt(-3);
1756  }
1757  }
1758 
1759  if (Token::simpleMatch(tokenizer.tokens(), "( 1 )") ||
1760  Token::simpleMatch(tokenizer.tokens(), "( 1 ||"))
1761  condition = "1";
1762  else if (Token::simpleMatch(tokenizer.tokens(), "( 0 )"))
1763  condition = "0";
1764 }
1765 
1766 bool Preprocessor::match_cfg_def(std::map<std::string, std::string> cfg, std::string def)
1767 {
1768  /*
1769  std::cout << "cfg: \"";
1770  for (std::map<std::string, std::string>::const_iterator it = cfg.begin(); it != cfg.end(); ++it)
1771  {
1772  std::cout << it->first;
1773  if (!it->second.empty())
1774  std::cout << "=" << it->second;
1775  std::cout << ";";
1776  }
1777  std::cout << "\" ";
1778  std::cout << "def: \"" << def << "\"\n";
1779  */
1780 
1781  simplifyVarMap(cfg, _settings);
1782  simplifyCondition(cfg, def, true);
1783 
1784  if (cfg.find(def) != cfg.end())
1785  return true;
1786 
1787  if (def == "0")
1788  return false;
1789 
1790  if (def == "1")
1791  return true;
1792 
1793  return false;
1794 }
1795 
1796 std::string Preprocessor::getcode(const std::string &filedata, const std::string &cfg, const std::string &filename)
1797 {
1798  // For the error report and preprocessor dump:
1799  // line number relative to current (included) file
1800  // (may decrease when popping back from an included file)
1801  unsigned int lineno = 0;
1802 
1803  std::ostringstream ret;
1804 
1805  bool match = true;
1806  std::list<bool> matching_ifdef;
1807  std::list<bool> matched_ifdef;
1808 
1809  // Create a map for the cfg for faster access to defines
1810  std::map<std::string, std::string> cfgmap(getcfgmap(cfg, &_settings, filename));
1811 
1812  std::stack<std::string> filenames;
1813  filenames.push(filename);
1814  std::stack<unsigned int> lineNumbers;
1815  std::istringstream istr(filedata);
1816  std::string line;
1817  directives.clear();
1818  while (std::getline(istr, line)) {
1819  ++lineno;
1820 
1821  if (line.empty()) {
1822  ret << '\n';
1823  continue;
1824  }
1825 
1826  // record directive for addons / checkers
1827  if ((line[0] == '#')
1828  && (line.compare(0, 6, "#line ") != 0)
1829  && (line.compare(0, 8, "#endfile") != 0)) {
1830  // for clarity, turn "#file ..." back into "#include ..."
1831  std::string orig_line = line;
1832  if (orig_line.compare(0, 6, "#file ")==0)
1833  orig_line.replace(1, 4, "include");
1834  // record directive and extra
1835  directives.push_back(Directive(filenames.top(), lineno,
1836  orig_line));
1837  }
1838 
1839  if (_settings.terminated())
1840  return "";
1841 
1842  if (line.compare(0, 11, "#pragma asm") == 0) {
1843  ret << "\n";
1844  bool found_end = false;
1845  while (getline(istr, line)) {
1846  if (line.compare(0, 14, "#pragma endasm") == 0) {
1847  found_end = true;
1848  break;
1849  }
1850 
1851  ret << "\n";
1852  }
1853  if (!found_end)
1854  break;
1855 
1856  if (line.find('=') != std::string::npos) {
1857  Tokenizer tokenizer(&_settings, _errorLogger);
1858  line.erase(0, sizeof("#pragma endasm"));
1859  std::istringstream tempIstr(line);
1860  tokenizer.tokenize(tempIstr, "", "", true);
1861  if (Token::Match(tokenizer.tokens(), "( %name% = %any% )")) {
1862  ret << "asm(" << tokenizer.tokens()->strAt(1) << ");";
1863  }
1864  }
1865 
1866  ret << "\n";
1867 
1868  continue;
1869  }
1870 
1871  const std::string def = getdef(line, true);
1872  const std::string ndef = getdef(line, false);
1873 
1874  const bool emptymatch = matching_ifdef.empty() || matched_ifdef.empty();
1875 
1876  if (line.compare(0, 8, "#define ") == 0) {
1877  match = true;
1878 
1879 
1880  typedef std::set<std::string>::const_iterator It;
1881  for (It it = _settings.userUndefs.begin(); it != _settings.userUndefs.end(); ++it) {
1882  const std::string::size_type symbolPos = line.find_first_not_of(' ', 8);
1883  if (symbolPos == std::string::npos)
1884  continue;
1885  const std::string::size_type undefMatchPos = line.find(*it, symbolPos);
1886  if (undefMatchPos == std::string::npos)
1887  continue;
1888  const std::string::size_type behindUndefPos = undefMatchPos + (*it).size();
1889  if ((line.size() == behindUndefPos) ||
1890  (line[behindUndefPos] == ' ') ||
1891  (line[behindUndefPos] == '(')) {
1892  match = false;
1893  break;
1894  }
1895  }
1896 
1897  if (match) {
1898  for (std::list<bool>::const_iterator it = matching_ifdef.begin(); it != matching_ifdef.end(); ++it) {
1899  if (!bool(*it)) {
1900  match = false;
1901  break;
1902  }
1903  }
1904  }
1905 
1906  if (match) {
1907  const std::string::size_type pos = line.find_first_of(" (", 8);
1908  if (pos == std::string::npos)
1909  cfgmap[line.substr(8)] = "";
1910  else if (line[pos] == ' ') {
1911  std::string value(line.substr(pos + 1));
1912  std::map<std::string, std::string>::const_iterator cfgpos = cfgmap.find(value);
1913  if (cfgpos != cfgmap.end())
1914  value = cfgpos->second;
1915  cfgmap[line.substr(8, pos - 8)] = value;
1916  } else
1917  cfgmap[line.substr(8, pos - 8)] = "";
1918  }
1919  }
1920 
1921  else if (line.compare(0, 7, "#undef ") == 0) {
1922  const std::string name(line.substr(7));
1923  cfgmap.erase(name);
1924  }
1925 
1926  else if (!emptymatch && line.compare(0, 7, "#elif !") == 0) {
1927  if (matched_ifdef.back()) {
1928  matching_ifdef.back() = false;
1929  } else {
1930  if (!match_cfg_def(cfgmap, ndef)) {
1931  matching_ifdef.back() = true;
1932  matched_ifdef.back() = true;
1933  }
1934  }
1935  }
1936 
1937  else if (!emptymatch && line.compare(0, 6, "#elif ") == 0) {
1938  if (matched_ifdef.back()) {
1939  matching_ifdef.back() = false;
1940  } else {
1941  if (match_cfg_def(cfgmap, def)) {
1942  matching_ifdef.back() = true;
1943  matched_ifdef.back() = true;
1944  }
1945  }
1946  }
1947 
1948  else if (line.compare(0,4,"#if ") == 0) {
1949  matching_ifdef.push_back(match_cfg_def(cfgmap, line.substr(4)));
1950  matched_ifdef.push_back(matching_ifdef.back());
1951  }
1952 
1953  else if (! def.empty()) {
1954  matching_ifdef.push_back(cfgmap.find(def) != cfgmap.end());
1955  matched_ifdef.push_back(matching_ifdef.back());
1956  }
1957 
1958  else if (! ndef.empty()) {
1959  matching_ifdef.push_back(cfgmap.find(ndef) == cfgmap.end());
1960  matched_ifdef.push_back(matching_ifdef.back());
1961  }
1962 
1963  else if (!emptymatch && line == "#else") {
1964  if (! matched_ifdef.empty())
1965  matching_ifdef.back() = ! matched_ifdef.back();
1966  }
1967 
1968  else if (line.compare(0, 6, "#endif") == 0) {
1969  if (! matched_ifdef.empty())
1970  matched_ifdef.pop_back();
1971  if (! matching_ifdef.empty())
1972  matching_ifdef.pop_back();
1973  }
1974 
1975  if (!line.empty() && line[0] == '#') {
1976  match = true;
1977  for (std::list<bool>::const_iterator it = matching_ifdef.begin(); it != matching_ifdef.end(); ++it) {
1978  if (!bool(*it)) {
1979  match = false;
1980  break;
1981  }
1982  }
1983  }
1984 
1985  // #error => return ""
1986  if (match && line.compare(0, 6, "#error") == 0) {
1987  if (!_settings.userDefines.empty() && !_settings.force) {
1988  error(Path::simplifyPath(filenames.top()), lineno, line);
1989  }
1990  return "";
1991  }
1992 
1993  if (!match && (line.compare(0, 8, "#define ") == 0 ||
1994  line.compare(0, 6, "#undef") == 0)) {
1995  // Remove define that is not part of this configuration
1996  line = "";
1997  } else if (line.compare(0, 7, "#file \"") == 0 ||
1998  line.compare(0, 8, "#endfile") == 0 ||
1999  line.compare(0, 8, "#define ") == 0 ||
2000  line.compare(0, 6, "#line ") == 0 ||
2001  line.compare(0, 6, "#undef") == 0) {
2002  // We must not remove #file tags or line numbers
2003  // are corrupted. File tags are removed by the tokenizer.
2004 
2005  // Keep location info updated
2006  if (line.compare(0, 7, "#file \"") == 0) {
2007  filenames.push(line.substr(7, line.size() - 8));
2008  lineNumbers.push(lineno);
2009  lineno = 0;
2010  } else if (line.compare(0, 8, "#endfile") == 0) {
2011  if (filenames.size() > 1U)
2012  filenames.pop();
2013 
2014  if (!lineNumbers.empty()) {
2015  lineno = lineNumbers.top();
2016  lineNumbers.pop();
2017  }
2018  }
2019  } else if (!match || line.compare(0, 1, "#") == 0) {
2020  // Remove #if, #else, #pragma etc, leaving only
2021  // #define, #undef, #file and #endfile. and also lines
2022  // which are not part of this configuration.
2023  line = "";
2024  }
2025 
2026  ret << line << "\n";
2027  }
2028 
2029  if (!validateCfg(ret.str(), cfg)) {
2030  return "";
2031  }
2032 
2033  return expandMacros(ret.str(), filename, cfg, _errorLogger);
2034 }
2035 
2036 void Preprocessor::error(const std::string &filename, unsigned int linenr, const std::string &msg)
2037 {
2038  std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
2039  if (!filename.empty()) {
2040  ErrorLogger::ErrorMessage::FileLocation loc(filename, linenr);
2041  locationList.push_back(loc);
2042  }
2045  msg,
2046  "preprocessorErrorDirective",
2047  false));
2048 }
2049 
2051 {
2052  std::string::size_type i = str.find_first_of("<\"");
2053  if (i == std::string::npos) {
2054  str = "";
2055  return NoHeader;
2056  }
2057 
2058  char c = str[i];
2059  if (c == '<')
2060  c = '>';
2061 
2062  std::string result;
2063  for (i = i + 1; i < str.length(); ++i) {
2064  if (str[i] == c)
2065  break;
2066 
2067  result.append(1, str[i]);
2068  }
2069 
2070  // Linux can't open include paths with \ separator, so fix them
2071  std::replace(result.begin(), result.end(), '\\', '/');
2072 
2073  str = result;
2074 
2075  return (c == '\"') ? UserHeader : SystemHeader;
2076 }
2077 
2078 /**
2079  * Try to open header
2080  * @param filename header name (in/out)
2081  * @param includePaths paths where to look for the file
2082  * @param filePath path to the header file
2083  * @param fin file input stream (in/out)
2084  * @return if file is opened then true is returned
2085  */
2086 static bool openHeader(std::string &filename, const std::list<std::string> &includePaths, const std::string &filePath, std::ifstream &fin)
2087 {
2088  fin.open((filePath + filename).c_str());
2089  if (fin.is_open()) {
2090  filename = filePath + filename;
2091  return true;
2092  }
2093 
2094  fin.open(filename.c_str());
2095  if (fin.is_open())
2096  return true;
2097 
2098  for (std::list<std::string>::const_iterator iter = includePaths.begin(); iter != includePaths.end(); ++iter) {
2099  const std::string nativePath(Path::toNativeSeparators(*iter));
2100  fin.open((nativePath + filename).c_str());
2101  if (fin.is_open()) {
2102  filename = nativePath + filename;
2103  return true;
2104  }
2105  fin.clear();
2106  }
2107 
2108  return false;
2109 }
2110 
2111 
2112 std::string Preprocessor::handleIncludes(const std::string &code, const std::string &filePath, const std::list<std::string> &includePaths, std::map<std::string,std::string> &defs, std::set<std::string> &pragmaOnce, std::list<std::string> includes)
2113 {
2114  std::string path;
2115  std::string::size_type sep_pos = filePath.find_last_of("\\/");
2116  if (sep_pos != std::string::npos)
2117  path = filePath.substr(0, 1 + sep_pos);
2118 
2119  // current #if indent level.
2120  std::stack<bool>::size_type indent = 0;
2121 
2122  // how deep does the #if match? this can never be bigger than "indent".
2123  std::stack<bool>::size_type indentmatch = 0;
2124 
2125  // has there been a true #if condition at the current indentmatch level?
2126  // then no more #elif or #else can be true before the #endif is seen.
2127  std::stack<bool> elseIsTrueStack;
2128 
2129  unsigned int linenr = 0;
2130 
2131  const std::set<std::string> &undefs = _settings.userUndefs;
2132 
2133  if (_errorLogger)
2134  _errorLogger->reportProgress(filePath, "Preprocessor (handleIncludes)", 0);
2135 
2136  std::ostringstream ostr;
2137  std::istringstream istr(code);
2138  std::string line;
2139  bool suppressCurrentCodePath = false;
2140  while (std::getline(istr,line)) {
2141  ++linenr;
2142 
2143  if (_settings.terminated())
2144  return "";
2145 
2146  // has there been a true #if condition at the current indentmatch level?
2147  // then no more #elif or #else can be true before the #endif is seen.
2148  while (elseIsTrueStack.size() != indentmatch + 1) {
2149  if (elseIsTrueStack.size() < indentmatch + 1) {
2150  elseIsTrueStack.push(true);
2151  } else {
2152  elseIsTrueStack.pop();
2153  }
2154  }
2155 
2156  if (elseIsTrueStack.empty()) {
2157  writeError(filePath, linenr, _errorLogger, "syntaxError", "Syntax error in preprocessor code");
2158  return "";
2159  }
2160 
2161  std::stack<bool>::reference elseIsTrue = elseIsTrueStack.top();
2162 
2163  if (line == "#pragma once") {
2164  pragmaOnce.insert(filePath);
2165  } else if (line.compare(0,7,"#ifdef ") == 0) {
2166  if (indent == indentmatch) {
2167  const std::string tag = getdef(line,true);
2168  if (defs.find(tag) != defs.end()) {
2169  elseIsTrue = false;
2170  indentmatch++;
2171  } else if (undefs.find(tag) != undefs.end()) {
2172  elseIsTrue = true;
2173  indentmatch++;
2174  suppressCurrentCodePath = true;
2175  }
2176  }
2177  ++indent;
2178 
2179  if (indent == indentmatch + 1)
2180  elseIsTrue = true;
2181  } else if (line.compare(0,8,"#ifndef ") == 0) {
2182  if (indent == indentmatch) {
2183  const std::string tag = getdef(line,false);
2184  if (defs.find(tag) == defs.end()) {
2185  elseIsTrue = false;
2186  indentmatch++;
2187  } else if (undefs.find(tag) != undefs.end()) {
2188  elseIsTrue = false;
2189  indentmatch++;
2190  suppressCurrentCodePath = false;
2191  }
2192  }
2193  ++indent;
2194 
2195  if (indent == indentmatch + 1)
2196  elseIsTrue = true;
2197 
2198  } else if (line.compare(0,4,"#if ") == 0) {
2199  if (!suppressCurrentCodePath && indent == indentmatch && match_cfg_def(defs, line.substr(4))) {
2200  elseIsTrue = false;
2201  indentmatch++;
2202  }
2203  ++indent;
2204 
2205  if (indent == indentmatch + 1)
2206  elseIsTrue = true; // this value doesn't matter when suppressCurrentCodePath is true
2207  } else if (line.compare(0,6,"#elif ") == 0 || line.compare(0,5,"#else") == 0) {
2208  if (!elseIsTrue) {
2209  if ((indentmatch > 0) && (indentmatch == indent)) {
2210  indentmatch = indent - 1;
2211  }
2212  } else {
2213  if ((indentmatch > 0) && (indentmatch == indent)) {
2214  indentmatch = indent - 1;
2215  } else if ((indent > 0) && indentmatch == indent - 1) {
2216  if (line.compare(0,5,"#else")==0 || match_cfg_def(defs,line.substr(6))) {
2217  indentmatch = indent;
2218  elseIsTrue = false;
2219  }
2220  }
2221  }
2222  } else if (line.compare(0, 6, "#endif") == 0) {
2223  if (indent > 0)
2224  --indent;
2225  if (indentmatch > indent || indent == 0) {
2226  indentmatch = indent;
2227  elseIsTrue = false;
2228  suppressCurrentCodePath = false;
2229  }
2230  } else if (indentmatch == indent) {
2231  if (!suppressCurrentCodePath && line.compare(0, 8, "#define ") == 0) {
2232  const unsigned int endOfDefine = 8;
2233  std::string::size_type endOfTag = line.find_first_of("( ", endOfDefine);
2234  std::string tag;
2235 
2236  // define a symbol
2237  if (endOfTag == std::string::npos) {
2238  tag = line.substr(endOfDefine);
2239  defs[tag] = "";
2240  } else {
2241  tag = line.substr(endOfDefine, endOfTag-endOfDefine);
2242 
2243  // define a function-macro
2244  if (line[endOfTag] == '(') {
2245  defs[tag] = "";
2246  }
2247  // define value
2248  else {
2249  ++endOfTag;
2250 
2251  const std::string& value = line.substr(endOfTag, line.size()-endOfTag);
2252 
2253  if (defs.find(value) != defs.end())
2254  defs[tag] = defs[value];
2255  else
2256  defs[tag] = value;
2257  }
2258  }
2259 
2260  if (undefs.find(tag) != undefs.end()) {
2261  defs.erase(tag);
2262  }
2263  }
2264 
2265  else if (!suppressCurrentCodePath && line.compare(0,7,"#undef ") == 0) {
2266  defs.erase(line.substr(7));
2267  }
2268 
2269  else if (!suppressCurrentCodePath && line.compare(0,9,"#include ")==0) {
2270  std::string filename(line.substr(9));
2271 
2272  const HeaderTypes headerType = getHeaderFileName(filename);
2273  if (headerType == NoHeader) {
2274  ostr << std::endl;
2275  continue;
2276  }
2277 
2278  // try to open file
2279  std::string filepath;
2280  if (headerType == UserHeader)
2281  filepath = path;
2282  std::ifstream fin;
2283  if (!openHeader(filename, includePaths, filepath, fin)) {
2285  linenr,
2286  filename,
2287  headerType
2288  );
2289  ostr << std::endl;
2290  continue;
2291  }
2292 
2293  // Prevent that files are recursively included
2294  if (std::find(includes.begin(), includes.end(), filename) != includes.end()) {
2295  ostr << std::endl;
2296  continue;
2297  }
2298 
2299  // #6913 - simplify Path to avoid strange recursion
2300  includes.push_back(Path::simplifyPath(filename));
2301 
2302  // Don't include header if it's already included and contains #pragma once
2303  if (pragmaOnce.find(filename) != pragmaOnce.end()) {
2304  ostr << std::endl;
2305  continue;
2306  }
2307 
2308  ostr << "#file \"" << filename << "\"\n"
2309  << handleIncludes(read(fin, filename), filename, includePaths, defs, pragmaOnce, includes) << std::endl
2310  << "#endfile\n";
2311  continue;
2312  }
2313 
2314  if (!suppressCurrentCodePath)
2315  ostr << line;
2316  }
2317 
2318  // A line has been read..
2319  ostr << "\n";
2320  }
2321 
2322  return ostr.str();
2323 }
2324 
2325 
2326 void Preprocessor::handleIncludes(std::string &code, const std::string &filePath, const std::list<std::string> &includePaths)
2327 {
2328  std::list<std::string> paths;
2329  std::string path = filePath;
2330  const std::string::size_type sep_pos = path.find_last_of("\\/");
2331  if (sep_pos != std::string::npos)
2332  path.erase(1 + sep_pos);
2333  paths.push_back(path);
2334  std::string::size_type pos = 0;
2335  std::string::size_type endfilePos = 0;
2336  if (code.compare(0,7U,"#file \"")==0) {
2337  const std::string::size_type start = code.find("#file \"" + filePath, 7U);
2338  if (start != std::string::npos)
2339  endfilePos = start;
2340  }
2341  std::set<std::string> handledFiles;
2342  while ((pos = code.find("#include", pos)) != std::string::npos) {
2343  if (_settings.terminated())
2344  return;
2345 
2346  // Accept only includes that are at the start of a line
2347  if (pos > 0 && code[pos-1] != '\n') {
2348  pos += 8; // length of "#include"
2349  continue;
2350  }
2351 
2352  // If endfile is encountered, we have moved to a next file in our stack,
2353  // so remove last path in our list.
2354  while (!paths.empty() && (endfilePos = code.find("\n#endfile", endfilePos)) != std::string::npos && endfilePos < pos) {
2355  paths.pop_back();
2356  endfilePos += 9; // size of #endfile
2357  }
2358 
2359  endfilePos = pos;
2360  std::string::size_type end = code.find('\n', pos);
2361  std::string filename = code.substr(pos, end - pos);
2362 
2363  // Remove #include clause
2364  code.erase(pos, end - pos);
2365 
2366  HeaderTypes headerType = getHeaderFileName(filename);
2367  if (headerType == NoHeader)
2368  continue;
2369 
2370  // filename contains now a file name e.g. "menu.h"
2371  std::string processedFile;
2372  std::string filepath;
2373  if (headerType == UserHeader && !paths.empty())
2374  filepath = paths.back();
2375  std::ifstream fin;
2376  const bool fileOpened(openHeader(filename, includePaths, filepath, fin));
2377 
2378  if (fileOpened) {
2379  filename = Path::simplifyPath(filename);
2380  std::string tempFile = filename;
2381  std::transform(tempFile.begin(), tempFile.end(), tempFile.begin(), tolowerWrapper);
2382  if (handledFiles.find(tempFile) != handledFiles.end()) {
2383  // We have processed this file already once, skip
2384  // it this time to avoid eternal loop.
2385  fin.close();
2386  continue;
2387  }
2388 
2389  handledFiles.insert(tempFile);
2390  processedFile = Preprocessor::read(fin, filename);
2391  fin.close();
2392  }
2393 
2394  if (!processedFile.empty()) {
2395  // Remove space characters that are after or before new line character
2396  processedFile = "#file \"" + Path::fromNativeSeparators(filename) + "\"\n" + processedFile + "\n#endfile";
2397  code.insert(pos, processedFile);
2398 
2399  path = filename;
2400  path.erase(1 + path.find_last_of("\\/"));
2401  paths.push_back(path);
2402  } else if (!fileOpened) {
2403  std::string f = filePath;
2404 
2405  // Determine line number of include
2406  unsigned int linenr = 1;
2407  unsigned int level = 0;
2408  for (std::string::size_type p = 1; p <= pos; ++p) {
2409  if (level == 0 && code[pos-p] == '\n')
2410  ++linenr;
2411  else if (code.compare(pos-p, 9, "#endfile\n") == 0) {
2412  ++level;
2413  } else if (code.compare(pos-p, 6, "#file ") == 0) {
2414  if (level == 0) {
2415  linenr--;
2416  const std::string::size_type pos1 = pos - p + 7;
2417  const std::string::size_type pos2 = code.find_first_of("\"\n", pos1);
2418  f = code.substr(pos1, (pos2 == std::string::npos) ? pos2 : (pos2 - pos1));
2419  break;
2420  }
2421  --level;
2422  }
2423  }
2424 
2426  linenr,
2427  filename,
2428  headerType);
2429  }
2430  }
2431 }
2432 
2433 // Report that include is missing
2434 void Preprocessor::missingInclude(const std::string &filename, unsigned int linenr, const std::string &header, HeaderTypes headerType)
2435 {
2436  const std::string fname = Path::fromNativeSeparators(filename);
2437  if (_settings.nomsg.isSuppressed("missingInclude", fname, linenr))
2438  return;
2439  if (headerType == SystemHeader && _settings.nomsg.isSuppressed("missingIncludeSystem", fname, linenr))
2440  return;
2441 
2442  if (headerType == SystemHeader)
2443  missingSystemIncludeFlag = true;
2444  else
2445  missingIncludeFlag = true;
2447 
2448  std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
2449  if (!filename.empty()) {
2451  loc.line = linenr;
2452  loc.setfile(Path::toNativeSeparators(filename));
2453  locationList.push_back(loc);
2454  }
2455  ErrorLogger::ErrorMessage errmsg(locationList, Severity::information,
2456  (headerType==SystemHeader) ?
2457  "Include file: <" + header + "> not found. Please note: Cppcheck does not need standard library headers to get proper results." :
2458  "Include file: \"" + header + "\" not found.",
2459  (headerType==SystemHeader) ? "missingIncludeSystem" : "missingInclude",
2460  false);
2461  errmsg.file0 = file0;
2462  _errorLogger->reportInfo(errmsg);
2463  }
2464 }
2465 
2466 /**
2467  * Skip string in line. A string begins and ends with either a &quot; or a &apos;
2468  * @param line the string
2469  * @param pos in=start position of string, out=end position of string
2470  */
2471 static void skipstring(const std::string &line, std::string::size_type &pos)
2472 {
2473  const char ch = line[pos];
2474 
2475  ++pos;
2476  while (pos < line.size() && line[pos] != ch) {
2477  if (line[pos] == '\\')
2478  ++pos;
2479  ++pos;
2480  }
2481 }
2482 
2483 /**
2484  * @brief get parameters from code. For example 'foo(1,2)' => '1','2'
2485  * @param line in: The code
2486  * @param pos in: Position to the '('. out: Position to the ')'
2487  * @param params out: The extracted parameters
2488  * @param numberOfNewlines out: number of newlines in the macro call
2489  * @param endFound out: was the end parentheses found?
2490  */
2491 static void getparams(const std::string &line,
2492  std::string::size_type &pos,
2493  std::vector<std::string> &params,
2494  unsigned int &numberOfNewlines,
2495  bool &endFound)
2496 {
2497  params.clear();
2498  numberOfNewlines = 0;
2499  endFound = false;
2500 
2501  if (line[pos] == ' ')
2502  pos++;
2503 
2504  if (line[pos] != '(')
2505  return;
2506 
2507  // parentheses level
2508  int parlevel = 0;
2509 
2510  // current parameter data
2511  std::string par;
2512 
2513  // scan for parameters..
2514  for (; pos < line.length(); ++pos) {
2515  // increase parentheses level
2516  if (line[pos] == '(') {
2517  ++parlevel;
2518  if (parlevel == 1)
2519  continue;
2520  }
2521 
2522  // decrease parentheses level
2523  else if (line[pos] == ')') {
2524  --parlevel;
2525  if (parlevel <= 0) {
2526  endFound = true;
2527  params.push_back(trim(par));
2528  break;
2529  }
2530  }
2531 
2532  // string
2533  else if (line[pos] == '\"' || line[pos] == '\'') {
2534  const std::string::size_type p = pos;
2535  skipstring(line, pos);
2536  if (pos == line.length())
2537  break;
2538  par += line.substr(p, pos + 1 - p);
2539  continue;
2540  }
2541 
2542  // count newlines. the expanded macro must have the same number of newlines
2543  else if (line[pos] == '\n') {
2544  ++numberOfNewlines;
2545  continue;
2546  }
2547 
2548  // new parameter
2549  if (parlevel == 1 && line[pos] == ',') {
2550  params.push_back(trim(par));
2551  par = "";
2552  }
2553 
2554  // spaces are only added if needed
2555  else if (line[pos] == ' ') {
2556  // Add space only if it is needed
2557  if (par.size() && std::isalnum((unsigned char)par.back())) {
2558  par += ' ';
2559  }
2560  }
2561 
2562  // add character to current parameter
2563  else if (parlevel >= 1 && line[pos] != Preprocessor::macroChar) {
2564  par.append(1, line[pos]);
2565  }
2566  }
2567 }
2568 
2569 /** @brief Class that the preprocessor uses when it expands macros. This class represents a preprocessor macro */
2571 private:
2572  /** tokens of this macro */
2574 
2575  /** macro parameters */
2576  std::vector<std::string> _params;
2577 
2578  /** macro definition in plain text */
2579  const std::string _macro;
2580 
2581  /** does this macro take a variable number of parameters? */
2583 
2584  /** The macro has parentheses but no parameters.. "AAA()" */
2585  bool _nopar;
2586 
2587  /** disabled assignment operator */
2588  void operator=(const PreprocessorMacro &);
2589 
2590  /** @brief expand inner macro */
2591  std::vector<std::string> expandInnerMacros(const std::vector<std::string> &params1,
2592  const std::map<std::string, PreprocessorMacro *> &macros) const {
2593  std::string innerMacroName;
2594 
2595  // Is there an inner macro..
2596  {
2597  const Token *tok = Token::findsimplematch(tokens(), ")");
2598  if (!Token::Match(tok, ") %name% ("))
2599  return params1;
2600  innerMacroName = tok->strAt(1);
2601  tok = tok->tokAt(3);
2602  unsigned int par = 0;
2603  while (Token::Match(tok, "%name% ,|)")) {
2604  tok = tok->tokAt(2);
2605  par++;
2606  }
2607  if (tok || par != params1.size())
2608  return params1;
2609  }
2610 
2611  std::vector<std::string> params2(params1);
2612 
2613  for (std::size_t ipar = 0; ipar < params1.size(); ++ipar) {
2614  const std::string s(innerMacroName + "(");
2615  const std::string param(params1[ipar]);
2616  if (param.compare(0,s.length(),s)==0 && param.back() == ')') {
2617  std::vector<std::string> innerparams;
2618  std::string::size_type pos = s.length() - 1;
2619  unsigned int num = 0;
2620  bool endFound = false;
2621  getparams(param, pos, innerparams, num, endFound);
2622  if (pos == param.length()-1 && num==0 && endFound && innerparams.size() == params1.size()) {
2623  // Is inner macro defined?
2624  std::map<std::string, PreprocessorMacro *>::const_iterator it = macros.find(innerMacroName);
2625  if (it != macros.end()) {
2626  // expand the inner macro
2627  const PreprocessorMacro *innerMacro = it->second;
2628 
2629  std::string innercode;
2630  std::map<std::string,PreprocessorMacro *> innermacros = macros;
2631  innermacros.erase(innerMacroName);
2632  innerMacro->code(innerparams, innermacros, innercode);
2633  params2[ipar] = innercode;
2634  }
2635  }
2636  }
2637  }
2638 
2639  return params2;
2640  }
2641 
2642 public:
2643  /**
2644  * @brief Constructor for PreprocessorMacro. This is the "setter"
2645  * for this class - everything is setup here.
2646  * @param [in] macro The code after define, until end of line,
2647  * e.g. "A(x) foo(x);"
2648  * @param [in] settings Current settings being used
2649  */
2650  PreprocessorMacro(const std::string &macro, const Settings* settings)
2651  : tokenlist(settings), _macro(macro) {
2652 
2653  // Tokenize the macro to make it easier to handle
2654  std::istringstream istr(macro);
2655  tokenlist.createTokens(istr);
2656 
2657  // initialize parameters to default values
2658  _variadic = _nopar = false;
2659 
2660  const std::string::size_type pos = macro.find_first_of(" (");
2661  if (pos != std::string::npos && macro[pos] == '(') {
2662  // Extract macro parameters
2663  if (Token::Match(tokens(), "%name% ( %name%")) {
2664  for (const Token *tok = tokens()->tokAt(2); tok; tok = tok->next()) {
2665  if (tok->str() == ")")
2666  break;
2667  if (Token::simpleMatch(tok, ". . . )")) {
2668  if (tok->previous()->str() == ",")
2669  _params.push_back("__VA_ARGS__");
2670  _variadic = true;
2671  break;
2672  }
2673  if (tok->isName())
2674  _params.push_back(tok->str());
2675  }
2676  }
2677 
2678  else if (Token::Match(tokens(), "%name% ( . . . )"))
2679  _variadic = true;
2680 
2681  else if (Token::Match(tokens(), "%name% ( )"))
2682  _nopar = true;
2683  }
2684  }
2685 
2686  /** return tokens of this macro */
2687  const Token *tokens() const {
2688  return tokenlist.front();
2689  }
2690 
2691  /** read parameters of this macro */
2692  const std::vector<std::string> &params() const {
2693  return _params;
2694  }
2695 
2696  /** check if this is macro has a variable number of parameters */
2697  bool variadic() const {
2698  return _variadic;
2699  }
2700 
2701  /** Check if this macro has parentheses but no parameters */
2702  bool nopar() const {
2703  return _nopar;
2704  }
2705 
2706  /** name of macro */
2707  const std::string &name() const {
2708  return tokens() ? tokens()->str() : emptyString;
2709  }
2710 
2711  /**
2712  * get expanded code for this macro
2713  * @param params2 macro parameters
2714  * @param macros macro definitions (recursion)
2715  * @param macrocode output string
2716  * @return true if the expanding was successful
2717  */
2718  bool code(const std::vector<std::string> &params2, const std::map<std::string, PreprocessorMacro *> &macros, std::string &macrocode) const {
2719  if (_nopar || (_params.empty() && _variadic)) {
2720  macrocode = _macro.substr(1 + _macro.find(')'));
2721  if (macrocode.empty())
2722  return true;
2723 
2724  std::string::size_type pos = 0;
2725  // Remove leading spaces
2726  if ((pos = macrocode.find_first_not_of(" ")) > 0)
2727  macrocode.erase(0, pos);
2728  // Remove ending newline
2729  if ((pos = macrocode.find_first_of("\r\n")) != std::string::npos)
2730  macrocode.erase(pos);
2731 
2732  // Replace "__VA_ARGS__" with parameters
2733  if (!_nopar) {
2734  std::string s;
2735  for (std::size_t i = 0; i < params2.size(); ++i) {
2736  if (i > 0)
2737  s += ",";
2738  s += params2[i];
2739  }
2740 
2741  pos = 0;
2742  while ((pos = macrocode.find("__VA_ARGS__", pos)) != std::string::npos) {
2743  macrocode.erase(pos, 11);
2744  macrocode.insert(pos, s);
2745  pos += s.length();
2746  }
2747  }
2748  }
2749 
2750  else if (_params.empty()) {
2751  std::string::size_type pos = _macro.find_first_of(" \"");
2752  if (pos == std::string::npos)
2753  macrocode = "";
2754  else {
2755  if (_macro[pos] == ' ')
2756  pos++;
2757  macrocode = _macro.substr(pos);
2758  if ((pos = macrocode.find_first_of("\r\n")) != std::string::npos)
2759  macrocode.erase(pos);
2760  }
2761  }
2762 
2763  else {
2764  const std::vector<std::string> givenparams = expandInnerMacros(params2, macros);
2765 
2766  const Token *tok = tokens();
2767  while (tok && tok->str() != ")")
2768  tok = tok->next();
2769  if (tok) {
2770  bool optcomma = false;
2771  while (nullptr != (tok = tok->next())) {
2772  std::string str = tok->str();
2773  if (str[0] == '#' || tok->isName()) {
2774  if (str == "##")
2775  continue;
2776 
2777  const bool stringify(str[0] == '#');
2778  if (stringify) {
2779  str = str.erase(0, 1);
2780  }
2781  for (std::size_t i = 0; i < _params.size(); ++i) {
2782  if (str == _params[i]) {
2783  if (_variadic &&
2784  (i == _params.size() - 1 ||
2785  (givenparams.size() + 2 == _params.size() && i + 1 == _params.size() - 1))) {
2786  str = "";
2787  for (std::size_t j = _params.size() - 1; j < givenparams.size(); ++j) {
2788  if (optcomma || j > _params.size() - 1)
2789  str += ",";
2790  optcomma = false;
2791  str += givenparams[j];
2792  }
2793  } else if (i >= givenparams.size()) {
2794  // Macro had more parameters than caller used.
2795  macrocode = "";
2796  return false;
2797  } else if (stringify) {
2798  const std::string &s(givenparams[i]);
2799  std::ostringstream ostr;
2800  ostr << "\"";
2801  for (std::string::size_type j = 0; j < s.size(); ++j) {
2802  if (s[j] == '\\' || s[j] == '\"')
2803  ostr << '\\';
2804  ostr << s[j];
2805  }
2806  str = ostr.str() + "\"";
2807  } else
2808  str = givenparams[i];
2809 
2810  break;
2811  }
2812  }
2813 
2814  // expand nopar macro
2815  if (tok->strAt(-1) != "##") {
2816  const std::map<std::string, PreprocessorMacro *>::const_iterator it = macros.find(str);
2817  if (it != macros.end() && it->second->_macro.find('(') == std::string::npos) {
2818  const std::string& macro = it->second->_macro;
2819  const std::string::size_type whitespacePos = macro.find(' ');
2820  if (whitespacePos != std::string::npos)
2821  str = macro.substr(whitespacePos);
2822  else
2823  str = "";
2824  }
2825  }
2826  }
2827  if (_variadic && tok->str() == "," && tok->next() && tok->next()->str() == "##") {
2828  optcomma = true;
2829  continue;
2830  }
2831  optcomma = false;
2832  // separate ++ and -- with space
2833  if (!str.empty() && !macrocode.empty() && (str[0] == '+' || str[0] == '-') && str[0] == macrocode[macrocode.size()-1U])
2834  macrocode += ' ';
2835  macrocode += str;
2836  if (Token::Match(tok, "%name% %name%|%num%") ||
2837  Token::Match(tok, "%num% %name%") ||
2838  Token::simpleMatch(tok, "> >"))
2839  macrocode += ' ';
2840  }
2841  }
2842  }
2843 
2844  return true;
2845  }
2846 };
2847 
2848 /**
2849  * Get data from a input string. This is an extended version of std::getline.
2850  * The std::getline only get a single line at a time. It can therefore happen that it
2851  * contains a partial statement. This function ensures that the returned data
2852  * doesn't end in the middle of a statement. The "getlines" name indicate that
2853  * this function will return multiple lines if needed.
2854  * @param istr input stream
2855  * @param line output data
2856  * @return success
2857  */
2858 static bool getlines(std::istream &istr, std::string &line)
2859 {
2860  if (!istr.good())
2861  return false;
2862  line = "";
2863  int parlevel = 0;
2864  bool directive = false;
2865  for (char ch = (char)istr.get(); istr.good(); ch = (char)istr.get()) {
2866  if (ch == '\'' || ch == '\"') {
2867  line += ch;
2868  char c = 0;
2869  while (istr.good() && c != ch) {
2870  if (c == '\\') {
2871  c = (char)istr.get();
2872  if (!istr.good())
2873  return true;
2874  line += c;
2875  }
2876 
2877  c = (char)istr.get();
2878  if (!istr.good())
2879  return true;
2880  if (c == '\n' && directive)
2881  return true;
2882  line += c;
2883  }
2884  continue;
2885  }
2886  if (ch == '(')
2887  ++parlevel;
2888  else if (ch == ')')
2889  --parlevel;
2890  else if (ch == '\n') {
2891  if (directive)
2892  return true;
2893 
2894  if (istr.peek() == '#') {
2895  line += ch;
2896  return true;
2897  }
2898  } else if (!directive && parlevel <= 0 && ch == ';') {
2899  line += ";";
2900  return true;
2901  }
2902 
2903  if (ch == '#' && line.empty())
2904  directive = true;
2905  line += ch;
2906  }
2907  return true;
2908 }
2909 
2910 bool Preprocessor::validateCfg(const std::string &code, const std::string &cfg)
2911 {
2912  const bool printInformation = _settings.isEnabled("information");
2913 
2914  // fill up "macros" with empty configuration macros
2915  std::set<std::string> macros;
2916  for (std::string::size_type pos = 0; pos < cfg.size();) {
2917  const std::string::size_type pos2 = cfg.find_first_of(";=", pos);
2918  if (pos2 == std::string::npos) {
2919  macros.insert(cfg.substr(pos));
2920  break;
2921  }
2922  if (cfg[pos2] == ';')
2923  macros.insert(cfg.substr(pos, pos2-pos));
2924  pos = cfg.find(';', pos2);
2925  if (pos != std::string::npos)
2926  ++pos;
2927  }
2928 
2929  // check if any empty macros are used in code
2930  for (std::set<std::string>::const_iterator it = macros.begin(); it != macros.end(); ++it) {
2931  const std::string &macro = *it;
2932  std::string::size_type pos = 0;
2933  while ((pos = code.find_first_of(std::string("#\"'")+macro[0], pos)) != std::string::npos) {
2934  const std::string::size_type pos1 = pos;
2935  const std::string::size_type pos2 = pos + macro.size();
2936  pos++;
2937 
2938  // skip string..
2939  if (code[pos1] == '\"' || code[pos1] == '\'') {
2940  while (pos < code.size() && code[pos] != code[pos1]) {
2941  if (code[pos] == '\\')
2942  ++pos;
2943  ++pos;
2944  }
2945  ++pos;
2946  }
2947 
2948  // skip preprocessor statement..
2949  else if (code[pos1] == '#') {
2950  if (pos1 == 0 || code[pos1-1] == '\n')
2951  pos = code.find('\n', pos);
2952  }
2953 
2954  // is macro used in code?
2955  else if (code.compare(pos1,macro.size(),macro) == 0) {
2956  if (pos1 > 0 && (std::isalnum((unsigned char)code[pos1-1U]) || code[pos1-1U] == '_'))
2957  continue;
2958  if (pos2 < code.size() && (std::isalnum((unsigned char)code[pos2]) || code[pos2] == '_'))
2959  continue;
2960  // macro is used in code, return false
2961  if (printInformation)
2962  validateCfgError(cfg, macro);
2963  return false;
2964  }
2965  }
2966  }
2967 
2968  return true;
2969 }
2970 
2971 void Preprocessor::validateCfgError(const std::string &cfg, const std::string &macro)
2972 {
2973  const std::string id = "ConfigurationNotChecked";
2974  std::list<ErrorLogger::ErrorMessage::FileLocation> locationList;
2976  locationList.push_back(loc);
2977  ErrorLogger::ErrorMessage errmsg(locationList, Severity::information, "Skipping configuration '" + cfg + "' since the value of '" + macro + "' is unknown. Use -D if you want to check it. You can use -U to skip it explicitly.", id, false);
2978  _errorLogger->reportInfo(errmsg);
2979 }
2980 
2981 std::string Preprocessor::expandMacros(const std::string &code, std::string filename, const std::string &cfg, ErrorLogger *errorLogger)
2982 {
2983  // Search for macros and expand them..
2984  // --------------------------------------------
2985 
2986  // Available macros (key=macroname, value=macro).
2987  std::map<std::string, PreprocessorMacro *> macros;
2988  const Settings settings;
2989 
2990  {
2991  // fill up "macros" with user defined macros
2992  const std::map<std::string,std::string> cfgmap(getcfgmap(cfg,nullptr,""));
2993  std::map<std::string, std::string>::const_iterator it;
2994  for (it = cfgmap.begin(); it != cfgmap.end(); ++it) {
2995  std::string s = it->first;
2996  if (!it->second.empty())
2997  s += " " + it->second;
2998  PreprocessorMacro *macro = new PreprocessorMacro(s, &settings);
2999  macros[it->first] = macro;
3000  }
3001  }
3002 
3003  // Current line number
3004  unsigned int linenr = 1;
3005 
3006  // linenr, filename
3007  std::stack< std::pair<unsigned int, std::string> > fileinfo;
3008 
3009  // output stream
3010  std::ostringstream ostr;
3011 
3012  // read code..
3013  std::istringstream istr(code);
3014  std::string line;
3015  while (getlines(istr, line)) {
3016  if (line.empty())
3017  continue;
3018 
3019  // Preprocessor directive
3020  if (line[0] == '#') {
3021  // defining a macro..
3022  if (line.compare(1, 7, "define ") == 0) {
3023  PreprocessorMacro *macro = new PreprocessorMacro(line.substr(8), &settings);
3024  if (macro->name().empty() || macro->name() == "NULL") {
3025  delete macro;
3026  } else if (macro->name() == "BOOST_FOREACH") {
3027  // BOOST_FOREACH is currently too complex to parse, so skip it.
3028  delete macro;
3029  } else {
3030  std::map<std::string, PreprocessorMacro *>::iterator it;
3031  it = macros.find(macro->name());
3032  if (it != macros.end())
3033  delete it->second;
3034  macros[macro->name()] = macro;
3035  }
3036  line = "\n";
3037  }
3038 
3039  // undefining a macro..
3040  else if (line.compare(1, 6, "undef ") == 0) {
3041  std::map<std::string, PreprocessorMacro *>::iterator it;
3042  it = macros.find(line.substr(7));
3043  if (it != macros.end()) {
3044  delete it->second;
3045  macros.erase(it);
3046  }
3047  line = "\n";
3048  }
3049 
3050  // entering a file, update position..
3051  else if (line.compare(1, 6, "file \"") == 0) {
3052  fileinfo.push(std::pair<unsigned int, std::string>(linenr, filename));
3053  filename = line.substr(7, line.length() - 8);
3054  linenr = 0;
3055  line += "\n";
3056  }
3057 
3058  // leaving a file, update position..
3059  else if (line == "#endfile") {
3060  if (!fileinfo.empty()) {
3061  linenr = fileinfo.top().first;
3062  filename = fileinfo.top().second;
3063  fileinfo.pop();
3064  }
3065  line += "\n";
3066  }
3067 
3068  // all other preprocessor directives are just replaced with a newline
3069  else
3070  line += "\n";
3071  }
3072 
3073  // expand macros..
3074  else {
3075  // Limit for each macro.
3076  // The limit specify a position in the "line" variable.
3077  // For a "recursive macro" where the expanded text contains
3078  // the macro again, the macro should not be expanded again.
3079  // The limits are used to prevent recursive expanding.
3080  // * When a macro is expanded its limit position is set to
3081  // the last expanded character.
3082  // * macros are only allowed to be expanded when the
3083  // the position is beyond the limit.
3084  // * The limit is relative to the end of the "line"
3085  // variable. Inserting and deleting text before the limit
3086  // without updating the limit is safe.
3087  // * when pos goes beyond a limit the limit needs to be
3088  // deleted because it is unsafe to insert/delete text
3089  // after the limit otherwise
3090  std::map<const PreprocessorMacro *, std::size_t> limits;
3091 
3092  // pos is the current position in line
3093  std::string::size_type pos = 0;
3094 
3095  // scan line to see if there are any macros to expand..
3096  unsigned int tmpLinenr = 0;
3097  while (pos < line.size()) {
3098  if (line[pos] == '\n')
3099  ++tmpLinenr;
3100 
3101  // skip strings..
3102  if (line[pos] == '\"' || line[pos] == '\'') {
3103  const char ch = line[pos];
3104 
3105  skipstring(line, pos);
3106  ++pos;
3107 
3108  if (pos >= line.size()) {
3109  writeError(Path::simplifyPath(filename),
3110  linenr + tmpLinenr,
3111  errorLogger,
3112  "noQuoteCharPair",
3113  std::string("No pair for character (") + ch + "). Can't process file. File is either invalid or unicode, which is currently not supported.");
3114 
3115  std::map<std::string, PreprocessorMacro *>::iterator it;
3116  for (it = macros.begin(); it != macros.end(); ++it)
3117  delete it->second;
3118  macros.clear();
3119  return "";
3120  }
3121 
3122  continue;
3123  }
3124 
3125  if (!std::isalpha((unsigned char)line[pos]) && line[pos] != '_')
3126  ++pos;
3127 
3128  // found an identifier..
3129  // the "while" is used in case the expanded macro will immediately call another macro
3130  while (pos < line.length() && (std::isalpha((unsigned char)line[pos]) || line[pos] == '_')) {
3131  // pos1 = start position of macro
3132  const std::string::size_type pos1 = pos++;
3133 
3134  // find the end of the identifier
3135  while (pos < line.size() && (std::isalnum((unsigned char)line[pos]) || line[pos] == '_'))
3136  ++pos;
3137 
3138  // get identifier
3139  const std::string id = line.substr(pos1, pos - pos1);
3140 
3141  // is there a macro with this name?
3142  std::map<std::string, PreprocessorMacro *>::const_iterator it;
3143  it = macros.find(id);
3144  if (it == macros.end())
3145  break; // no macro with this name exist
3146 
3147  const PreprocessorMacro * const macro = it->second;
3148 
3149  // check that pos is within allowed limits for this
3150  // macro
3151  {
3152  const std::map<const PreprocessorMacro *, std::size_t>::const_iterator it2 = limits.find(macro);
3153  if (it2 != limits.end() && pos <= line.length() - it2->second)
3154  break;
3155  }
3156 
3157  // get parameters from line..
3158  if (macro->params().size() && pos >= line.length())
3159  break;
3160  std::vector<std::string> params;
3161  std::string::size_type pos2 = pos;
3162 
3163  // number of newlines within macro use
3164  unsigned int numberOfNewlines = 0;
3165 
3166  // if the macro has parentheses, get parameters
3167  if (macro->variadic() || macro->nopar() || macro->params().size()) {
3168  // is the end parentheses found?
3169  bool endFound = false;
3170 
3171  getparams(line,pos2,params,numberOfNewlines,endFound);
3172 
3173  // something went wrong so bail out
3174  if (!endFound)
3175  break;
3176  }
3177 
3178  // Just an empty parameter => clear
3179  if (params.size() == 1 && params[0] == "")
3180  params.clear();
3181 
3182  // Check that it's the same number of parameters..
3183  if (!macro->variadic() && params.size() != macro->params().size())
3184  break;
3185 
3186  // Create macro code..
3187  std::string tempMacro;
3188  if (!macro->code(params, macros, tempMacro)) {
3189  // Syntax error in code
3190  writeError(filename,
3191  linenr + tmpLinenr,
3192  errorLogger,
3193  "syntaxError",
3194  std::string("Syntax error. Not enough parameters for macro '") + macro->name() + "'.");
3195 
3196  std::map<std::string, PreprocessorMacro *>::iterator iter;
3197  for (iter = macros.begin(); iter != macros.end(); ++iter)
3198  delete iter->second;
3199  macros.clear();
3200  return "";
3201  }
3202 
3203  // make sure number of newlines remain the same..
3204  std::string macrocode(std::string(numberOfNewlines, '\n') + tempMacro);
3205 
3206  // Insert macro code..
3207  if (macro->variadic() || macro->nopar() || !macro->params().empty())
3208  ++pos2;
3209 
3210  // Remove old limits
3211  for (std::map<const PreprocessorMacro *, std::size_t>::iterator iter = limits.begin();
3212  iter != limits.end();) {
3213  if ((line.length() - pos1) < iter->second) {
3214  // We have gone past this limit, so just delete it
3215  limits.erase(iter++);
3216  } else {
3217  ++iter;
3218  }
3219  }
3220 
3221  // don't allow this macro to be expanded again before pos2
3222  limits[macro] = line.length() - pos2;
3223 
3224  // erase macro
3225  line.erase(pos1, pos2 - pos1);
3226 
3227  // Don't glue this macro into variable or number after it
3228  if (!line.empty() && (std::isalnum((unsigned char)line[pos1]) || line[pos1] == '_'))
3229  macrocode.append(1,' ');
3230 
3231  // insert macrochar before each symbol/nr/operator
3232  bool str = false;
3233  bool chr = false;
3234  for (std::size_t i = 0U; i < macrocode.size(); ++i) {
3235  if (macrocode[i] == '\\') {
3236  i++;
3237  continue;
3238  } else if (macrocode[i] == '\"')
3239  str = !str;
3240  else if (macrocode[i] == '\'')
3241  chr = !chr;
3242  else if (str || chr)
3243  continue;
3244  else if (macrocode[i] == '.') { // 5. / .5
3245  if ((i > 0U && std::isdigit((unsigned char)macrocode[i-1])) ||
3246  (i+1 < macrocode.size() && std::isdigit((unsigned char)macrocode[i+1]))) {
3247  if (i > 0U && !std::isdigit((unsigned char)macrocode[i-1])) {
3248  macrocode.insert(i, 1U, macroChar);
3249  i++;
3250  }
3251  i++;
3252  if (i<macrocode.size() && std::isdigit((unsigned char)macrocode[i]))
3253  i++;
3254  if (i+1U < macrocode.size() &&
3255  (macrocode[i] == 'e' || macrocode[i] == 'E') &&
3256  (macrocode[i+1] == '+' || macrocode[i+1] == '-')) {
3257  i+=2;
3258  }
3259  }
3260  } else if (std::isalnum((unsigned char)macrocode[i]) || macrocode[i] == '_') {
3261  if ((i > 0U) &&
3262  (!std::isalnum((unsigned char)macrocode[i-1])) &&
3263  (macrocode[i-1] != '_') &&
3264  (macrocode[i-1] != macroChar)) {
3265  macrocode.insert(i, 1U, macroChar);
3266  }
3267 
3268  // 1e-7 / 1e+7
3269  if (i+3U < macrocode.size() &&
3270  (std::isdigit((unsigned char)macrocode[i]) || macrocode[i]=='.') &&
3271  (macrocode[i+1] == 'e' || macrocode[i+1] == 'E') &&
3272  (macrocode[i+2] == '-' || macrocode[i+2] == '+') &&
3273  std::isdigit((unsigned char)macrocode[i+3])) {
3274  i += 3U;
3275  }
3276 
3277  // 1.f / 1.e7
3278  if (i+2U < macrocode.size() &&
3279  std::isdigit((unsigned char)macrocode[i]) &&
3280  macrocode[i+1] == '.' &&
3281  std::isalpha((unsigned char)macrocode[i+2])) {
3282  i += 2U;
3283  if (i+2U < macrocode.size() &&
3284  (macrocode[i+0] == 'e' || macrocode[i+0] == 'E') &&
3285  (macrocode[i+1] == '-' || macrocode[i+1] == '+') &&
3286  std::isdigit((unsigned char)macrocode[i+2])) {
3287  i += 2U;
3288  }
3289  }
3290  }
3291  }
3292  line.insert(pos1, macroChar + macrocode);
3293 
3294  // position = start position.
3295  pos = pos1;
3296  }
3297  }
3298  }
3299 
3300  // the line has been processed in various ways. Now add it to the output stream
3301  ostr << line;
3302 
3303  // update linenr
3304  for (std::string::size_type p = 0; p < line.length(); ++p) {
3305  if (line[p] == '\n')
3306  ++linenr;
3307  }
3308  }
3309 
3310  for (std::map<std::string, PreprocessorMacro *>::iterator it = macros.begin(); it != macros.end(); ++it)
3311  delete it->second;
3312  macros.clear();
3313 
3314  return ostr.str();
3315 }
3316 
3317 
3318 void Preprocessor::getErrorMessages(ErrorLogger *errorLogger, const Settings *settings)
3319 {
3320  Settings settings2(*settings);
3321  Preprocessor preprocessor(settings2, errorLogger);
3322  settings2.checkConfiguration=true;
3323  preprocessor.missingInclude("", 1, "", UserHeader);
3324  preprocessor.missingInclude("", 1, "", SystemHeader);
3325  preprocessor.validateCfgError("X", "X");
3326  preprocessor.error("", 1, "#error message"); // #error ..
3327 }
3328 
3329 void Preprocessor::dump(std::ostream &out) const
3330 {
3331  // Create a xml directive dump.
3332  // The idea is not that this will be readable for humans. It's a
3333  // data dump that 3rd party tools could load and get useful info from.
3334  std::list<Directive>::const_iterator it;
3335 
3336  out << " <directivelist>" << std::endl;
3337 
3338  for (it = directives.begin(); it != directives.end(); ++it) {
3339  out << " <directive "
3340  << "file=\"" << it->file << "\" "
3341  << "linenr=\"" << it->linenr << "\" "
3342  // str might contain characters such as '"', '<' or '>' which
3343  // could result in invalid XML, so run it through toxml().
3344  << "str=\"" << ErrorLogger::toxml(it->str) << "\"/>" << std::endl;
3345  }
3346  out << " </directivelist>" << std::endl;
3347 }