ebpf.c 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953
  1. // SPDX-License-Identifier: GPL-3.0-or-later
  2. #include <sys/time.h>
  3. #include <sys/resource.h>
  4. #include <ifaddrs.h>
  5. #include "ebpf.h"
  6. #include "ebpf_socket.h"
  7. /*****************************************************************
  8. *
  9. * FUNCTIONS USED BY NETDATA
  10. *
  11. *****************************************************************/
  12. // callback required by eval()
  13. int health_variable_lookup(const char *variable, uint32_t hash, struct rrdcalc *rc, calculated_number *result)
  14. {
  15. UNUSED(variable);
  16. UNUSED(hash);
  17. UNUSED(rc);
  18. UNUSED(result);
  19. return 0;
  20. };
  21. void send_statistics(const char *action, const char *action_result, const char *action_data)
  22. {
  23. UNUSED(action);
  24. UNUSED(action_result);
  25. UNUSED(action_data);
  26. }
  27. // callbacks required by popen()
  28. void signals_block(void){};
  29. void signals_unblock(void){};
  30. void signals_reset(void){};
  31. // required by get_system_cpus()
  32. char *netdata_configured_host_prefix = "";
  33. // callback required by fatal()
  34. void netdata_cleanup_and_exit(int ret)
  35. {
  36. exit(ret);
  37. }
  38. // ----------------------------------------------------------------------
  39. /*****************************************************************
  40. *
  41. * GLOBAL VARIABLES
  42. *
  43. *****************************************************************/
  44. char *ebpf_plugin_dir = PLUGINS_DIR;
  45. char *ebpf_user_config_dir = CONFIG_DIR;
  46. char *ebpf_stock_config_dir = LIBCONFIG_DIR;
  47. static char *ebpf_configured_log_dir = LOG_DIR;
  48. int update_every = 1;
  49. static int thread_finished = 0;
  50. int close_ebpf_plugin = 0;
  51. struct config collector_config = { .first_section = NULL,
  52. .last_section = NULL,
  53. .mutex = NETDATA_MUTEX_INITIALIZER,
  54. .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
  55. .rwlock = AVL_LOCK_INITIALIZER } };
  56. int running_on_kernel = 0;
  57. char kernel_string[64];
  58. int ebpf_nprocs;
  59. static int isrh;
  60. uint32_t finalized_threads = 1;
  61. pthread_mutex_t lock;
  62. pthread_mutex_t collect_data_mutex;
  63. pthread_cond_t collect_data_cond_var;
  64. ebpf_module_t ebpf_modules[] = {
  65. { .thread_name = "process", .config_name = "process", .enabled = 0, .start_routine = ebpf_process_thread,
  66. .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
  67. .optional = 0 },
  68. { .thread_name = "socket", .config_name = "socket", .enabled = 0, .start_routine = ebpf_socket_thread,
  69. .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
  70. .optional = 0 },
  71. { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1,
  72. .global_charts = 0, .apps_charts = 1, .mode = MODE_ENTRY,
  73. .optional = 0 },
  74. };
  75. // Link with apps.plugin
  76. ebpf_process_stat_t *global_process_stat = NULL;
  77. //Network viewer
  78. ebpf_network_viewer_options_t network_viewer_opt;
  79. /*****************************************************************
  80. *
  81. * FUNCTIONS USED TO CLEAN MEMORY AND OPERATE SYSTEM FILES
  82. *
  83. *****************************************************************/
  84. /**
  85. * Clean port Structure
  86. *
  87. * Clean the allocated list.
  88. *
  89. * @param clean the list that will be cleaned
  90. */
  91. void clean_port_structure(ebpf_network_viewer_port_list_t **clean)
  92. {
  93. ebpf_network_viewer_port_list_t *move = *clean;
  94. while (move) {
  95. ebpf_network_viewer_port_list_t *next = move->next;
  96. freez(move->value);
  97. freez(move);
  98. move = next;
  99. }
  100. *clean = NULL;
  101. }
  102. /**
  103. * Clean IP structure
  104. *
  105. * Clean the allocated list.
  106. *
  107. * @param clean the list that will be cleaned
  108. */
  109. static void clean_ip_structure(ebpf_network_viewer_ip_list_t **clean)
  110. {
  111. ebpf_network_viewer_ip_list_t *move = *clean;
  112. while (move) {
  113. ebpf_network_viewer_ip_list_t *next = move->next;
  114. freez(move);
  115. move = next;
  116. }
  117. *clean = NULL;
  118. }
  119. /**
  120. * Clean Loaded Events
  121. *
  122. * This function cleans the events previous loaded on Linux.
  123. void clean_loaded_events()
  124. {
  125. int event_pid;
  126. for (event_pid = 0; ebpf_modules[event_pid].probes; event_pid++)
  127. clean_kprobe_events(NULL, (int)ebpf_modules[event_pid].thread_id, ebpf_modules[event_pid].probes);
  128. }
  129. */
  130. /**
  131. * Close the collector gracefully
  132. *
  133. * @param sig is the signal number used to close the collector
  134. */
  135. static void ebpf_exit(int sig)
  136. {
  137. close_ebpf_plugin = 1;
  138. // When both threads were not finished case I try to go in front this address, the collector will crash
  139. if (!thread_finished) {
  140. return;
  141. }
  142. freez(global_process_stat);
  143. /*
  144. int ret = fork();
  145. if (ret < 0) // error
  146. error("Cannot fork(), so I won't be able to clean %skprobe_events", NETDATA_DEBUGFS);
  147. else if (!ret) { // child
  148. int i;
  149. for (i = getdtablesize(); i >= 0; --i)
  150. close(i);
  151. int fd = open("/dev/null", O_RDWR, 0);
  152. if (fd != -1) {
  153. dup2(fd, STDIN_FILENO);
  154. dup2(fd, STDOUT_FILENO);
  155. dup2(fd, STDERR_FILENO);
  156. }
  157. if (fd > 2)
  158. close(fd);
  159. int sid = setsid();
  160. if (sid >= 0) {
  161. debug(D_EXIT, "Wait for father %d die", getpid());
  162. sleep_usec(200000); // Sleep 200 miliseconds to father dies.
  163. clean_loaded_events();
  164. } else {
  165. error("Cannot become session id leader, so I won't try to clean kprobe_events.\n");
  166. }
  167. } else { // parent
  168. exit(0);
  169. }
  170. */
  171. exit(sig);
  172. }
  173. /*****************************************************************
  174. *
  175. * FUNCTIONS TO CREATE CHARTS
  176. *
  177. *****************************************************************/
  178. /**
  179. * Get a value from a structure.
  180. *
  181. * @param basis it is the first address of the structure
  182. * @param offset it is the offset of the data you want to access.
  183. * @return
  184. */
  185. collected_number get_value_from_structure(char *basis, size_t offset)
  186. {
  187. collected_number *value = (collected_number *)(basis + offset);
  188. collected_number ret = (collected_number)llabs(*value);
  189. // this reset is necessary to avoid keep a constant value while processing is not executing a task
  190. *value = 0;
  191. return ret;
  192. }
  193. /**
  194. * Write begin command on standard output
  195. *
  196. * @param family the chart family name
  197. * @param name the chart name
  198. */
  199. void write_begin_chart(char *family, char *name)
  200. {
  201. printf("BEGIN %s.%s\n", family, name);
  202. }
  203. /**
  204. * Write END command on stdout.
  205. */
  206. inline void write_end_chart()
  207. {
  208. printf("END\n");
  209. }
  210. /**
  211. * Write set command on standard output
  212. *
  213. * @param dim the dimension name
  214. * @param value the value for the dimension
  215. */
  216. void write_chart_dimension(char *dim, long long value)
  217. {
  218. int ret = printf("SET %s = %lld\n", dim, value);
  219. UNUSED(ret);
  220. }
  221. /**
  222. * Call the necessary functions to create a chart.
  223. *
  224. * @param name the chart name
  225. * @param family the chart family
  226. * @param move the pointer with the values that will be published
  227. * @param end the number of values that will be written on standard output
  228. *
  229. * @return It returns a variable tha maps the charts that did not have zero values.
  230. */
  231. void write_count_chart(char *name, char *family, netdata_publish_syscall_t *move, uint32_t end)
  232. {
  233. write_begin_chart(family, name);
  234. uint32_t i = 0;
  235. while (move && i < end) {
  236. write_chart_dimension(move->name, move->ncall);
  237. move = move->next;
  238. i++;
  239. }
  240. write_end_chart();
  241. }
  242. /**
  243. * Call the necessary functions to create a chart.
  244. *
  245. * @param name the chart name
  246. * @param family the chart family
  247. * @param move the pointer with the values that will be published
  248. * @param end the number of values that will be written on standard output
  249. */
  250. void write_err_chart(char *name, char *family, netdata_publish_syscall_t *move, int end)
  251. {
  252. write_begin_chart(family, name);
  253. int i = 0;
  254. while (move && i < end) {
  255. write_chart_dimension(move->name, move->nerr);
  256. move = move->next;
  257. i++;
  258. }
  259. write_end_chart();
  260. }
  261. /**
  262. * Call the necessary functions to create a chart.
  263. *
  264. * @param family the chart family
  265. * @param move the pointer with the values that will be published
  266. *
  267. * @return It returns a variable tha maps the charts that did not have zero values.
  268. */
  269. void write_io_chart(char *chart, char *family, char *dwrite, char *dread, netdata_publish_vfs_common_t *pvc)
  270. {
  271. write_begin_chart(family, chart);
  272. write_chart_dimension(dwrite, (long long)pvc->write);
  273. write_chart_dimension(dread, (long long)pvc->read);
  274. write_end_chart();
  275. }
  276. /**
  277. * Write chart cmd on standard output
  278. *
  279. * @param type the chart type
  280. * @param id the chart id
  281. * @param title the chart title
  282. * @param units the units label
  283. * @param family the group name used to attach the chart on dashaboard
  284. * @param charttype the chart type
  285. * @param order the chart order
  286. */
  287. void ebpf_write_chart_cmd(char *type, char *id, char *title, char *units, char *family, char *charttype, int order)
  288. {
  289. printf("CHART %s.%s '' '%s' '%s' '%s' '' %s %d %d\n",
  290. type,
  291. id,
  292. title,
  293. units,
  294. family,
  295. charttype,
  296. order,
  297. update_every);
  298. }
  299. /**
  300. * Write the dimension command on standard output
  301. *
  302. * @param n the dimension name
  303. * @param d the dimension information
  304. */
  305. void ebpf_write_global_dimension(char *n, char *d)
  306. {
  307. printf("DIMENSION %s %s absolute 1 1\n", n, d);
  308. }
  309. /**
  310. * Call ebpf_write_global_dimension to create the dimensions for a specific chart
  311. *
  312. * @param ptr a pointer to a structure of the type netdata_publish_syscall_t
  313. * @param end the number of dimensions for the structure ptr
  314. */
  315. void ebpf_create_global_dimension(void *ptr, int end)
  316. {
  317. netdata_publish_syscall_t *move = ptr;
  318. int i = 0;
  319. while (move && i < end) {
  320. ebpf_write_global_dimension(move->name, move->dimension);
  321. move = move->next;
  322. i++;
  323. }
  324. }
  325. /**
  326. * Call write_chart_cmd to create the charts
  327. *
  328. * @param type the chart type
  329. * @param id the chart id
  330. * @param units the axis label
  331. * @param family the group name used to attach the chart on dashaboard
  332. * @param order the order number of the specified chart
  333. * @param ncd a pointer to a function called to create dimensions
  334. * @param move a pointer for a structure that has the dimensions
  335. * @param end number of dimensions for the chart created
  336. */
  337. void ebpf_create_chart(char *type,
  338. char *id,
  339. char *title,
  340. char *units,
  341. char *family,
  342. int order,
  343. void (*ncd)(void *, int),
  344. void *move,
  345. int end)
  346. {
  347. ebpf_write_chart_cmd(type, id, title, units, family, "line", order);
  348. ncd(move, end);
  349. }
  350. /**
  351. * Create charts on apps submenu
  352. *
  353. * @param id the chart id
  354. * @param title the value displayed on vertical axis.
  355. * @param units the value displayed on vertical axis.
  356. * @param family Submenu that the chart will be attached on dashboard.
  357. * @param order the chart order
  358. * @param root structure used to create the dimensions.
  359. */
  360. void ebpf_create_charts_on_apps(char *id, char *title, char *units, char *family, int order, struct target *root)
  361. {
  362. struct target *w;
  363. ebpf_write_chart_cmd(NETDATA_APPS_FAMILY, id, title, units, family, "stacked", order);
  364. for (w = root; w; w = w->next) {
  365. if (unlikely(w->exposed))
  366. fprintf(stdout, "DIMENSION %s '' absolute 1 1\n", w->name);
  367. }
  368. }
  369. /*****************************************************************
  370. *
  371. * FUNCTIONS TO DEFINE OPTIONS
  372. *
  373. *****************************************************************/
  374. /**
  375. * Define labels used to generate charts
  376. *
  377. * @param is structure with information about number of calls made for a function.
  378. * @param pio structure used to generate charts.
  379. * @param dim a pointer for the dimensions name
  380. * @param name a pointer for the tensor with the name of the functions.
  381. * @param end the number of elements in the previous 4 arguments.
  382. */
  383. void ebpf_global_labels(netdata_syscall_stat_t *is, netdata_publish_syscall_t *pio, char **dim, char **name, int end)
  384. {
  385. int i;
  386. netdata_syscall_stat_t *prev = NULL;
  387. netdata_publish_syscall_t *publish_prev = NULL;
  388. for (i = 0; i < end; i++) {
  389. if (prev) {
  390. prev->next = &is[i];
  391. }
  392. prev = &is[i];
  393. pio[i].dimension = dim[i];
  394. pio[i].name = name[i];
  395. if (publish_prev) {
  396. publish_prev->next = &pio[i];
  397. }
  398. publish_prev = &pio[i];
  399. }
  400. }
  401. /**
  402. * Define thread mode for all ebpf program.
  403. *
  404. * @param lmode the mode that will be used for them.
  405. */
  406. static inline void ebpf_set_thread_mode(netdata_run_mode_t lmode)
  407. {
  408. int i;
  409. for (i = 0; ebpf_modules[i].thread_name; i++) {
  410. ebpf_modules[i].mode = lmode;
  411. }
  412. }
  413. /**
  414. * Enable specific charts selected by user.
  415. *
  416. * @param em the structure that will be changed
  417. * @param enable the status about the apps charts.
  418. */
  419. static inline void ebpf_enable_specific_chart(struct ebpf_module *em, int enable)
  420. {
  421. em->enabled = 1;
  422. if (!enable) {
  423. em->apps_charts = 1;
  424. }
  425. em->global_charts = 1;
  426. }
  427. /**
  428. * Enable all charts
  429. *
  430. * @param apps what is the current status of apps
  431. */
  432. static inline void ebpf_enable_all_charts(int apps)
  433. {
  434. int i;
  435. for (i = 0; ebpf_modules[i].thread_name; i++) {
  436. ebpf_enable_specific_chart(&ebpf_modules[i], apps);
  437. }
  438. }
  439. /**
  440. * Enable the specified chart group
  441. *
  442. * @param idx the index of ebpf_modules that I am enabling
  443. * @param disable_apps should I keep apps charts?
  444. */
  445. static inline void ebpf_enable_chart(int idx, int disable_apps)
  446. {
  447. int i;
  448. for (i = 0; ebpf_modules[i].thread_name; i++) {
  449. if (i == idx) {
  450. ebpf_enable_specific_chart(&ebpf_modules[i], disable_apps);
  451. break;
  452. }
  453. }
  454. }
  455. /**
  456. * Disable APPs
  457. *
  458. * Disable charts for apps loading only global charts.
  459. */
  460. static inline void ebpf_disable_apps()
  461. {
  462. int i;
  463. for (i = 0; ebpf_modules[i].thread_name; i++) {
  464. ebpf_modules[i].apps_charts = 0;
  465. }
  466. }
  467. /**
  468. * Print help on standard error for user knows how to use the collector.
  469. */
  470. void ebpf_print_help()
  471. {
  472. const time_t t = time(NULL);
  473. struct tm ct;
  474. struct tm *test = localtime_r(&t, &ct);
  475. int year;
  476. if (test)
  477. year = ct.tm_year;
  478. else
  479. year = 0;
  480. fprintf(stderr,
  481. "\n"
  482. " Netdata ebpf.plugin %s\n"
  483. " Copyright (C) 2016-%d Costa Tsaousis <costa@tsaousis.gr>\n"
  484. " Released under GNU General Public License v3 or later.\n"
  485. " All rights reserved.\n"
  486. "\n"
  487. " This program is a data collector plugin for netdata.\n"
  488. "\n"
  489. " Available command line options:\n"
  490. "\n"
  491. " SECONDS set the data collection frequency.\n"
  492. "\n"
  493. " --help or -h show this help.\n"
  494. "\n"
  495. " --version or -v show software version.\n"
  496. "\n"
  497. " --global or -g disable charts per application.\n"
  498. "\n"
  499. " --all or -a Enable all chart groups (global and apps), unless -g is also given.\n"
  500. "\n"
  501. " --net or -n Enable network viewer charts.\n"
  502. "\n"
  503. " --process or -p Enable charts related to process run time.\n"
  504. "\n"
  505. " --return or -r Run the collector in return mode.\n"
  506. "\n",
  507. VERSION,
  508. (year >= 116) ? year + 1900 : 2020);
  509. }
  510. /*****************************************************************
  511. *
  512. * AUXILIAR FUNCTIONS USED DURING INITIALIZATION
  513. *
  514. *****************************************************************/
  515. /**
  516. * Is ip inside the range
  517. *
  518. * Check if the ip is inside a IP range
  519. *
  520. * @param rfirst the first ip address of the range
  521. * @param rlast the last ip address of the range
  522. * @param cmpfirst the first ip to compare
  523. * @param cmplast the last ip to compare
  524. * @param family the IP family
  525. *
  526. * @return It returns 1 if the IP is inside the range and 0 otherwise
  527. */
  528. static int is_ip_inside_range(union netdata_ip_t *rfirst, union netdata_ip_t *rlast,
  529. union netdata_ip_t *cmpfirst, union netdata_ip_t *cmplast, int family)
  530. {
  531. if (family == AF_INET) {
  532. if (ntohl(rfirst->addr32[0]) <= ntohl(cmpfirst->addr32[0]) &&
  533. ntohl(rlast->addr32[0]) >= ntohl(cmplast->addr32[0]))
  534. return 1;
  535. } else {
  536. if (memcmp(rfirst->addr8, cmpfirst->addr8, sizeof(union netdata_ip_t)) <= 0 &&
  537. memcmp(rlast->addr8, cmplast->addr8, sizeof(union netdata_ip_t)) >= 0) {
  538. return 1;
  539. }
  540. }
  541. return 0;
  542. }
  543. /**
  544. * Fill IP list
  545. *
  546. * @param out a pointer to the link list.
  547. * @param in the structure that will be linked.
  548. */
  549. static inline void fill_ip_list(ebpf_network_viewer_ip_list_t **out, ebpf_network_viewer_ip_list_t *in, char *table)
  550. {
  551. #ifndef NETDATA_INTERNAL_CHECKS
  552. UNUSED(table);
  553. #endif
  554. if (likely(*out)) {
  555. ebpf_network_viewer_ip_list_t *move = *out, *store = *out;
  556. while (move) {
  557. if (in->ver == move->ver && is_ip_inside_range(&move->first, &move->last, &in->first, &in->last, in->ver)) {
  558. info("The range/value (%s) is inside the range/value (%s) already inserted, it will be ignored.",
  559. in->value, move->value);
  560. freez(in->value);
  561. freez(in);
  562. return;
  563. }
  564. store = move;
  565. move = move->next;
  566. }
  567. store->next = in;
  568. } else {
  569. *out = in;
  570. }
  571. #ifdef NETDATA_INTERNAL_CHECKS
  572. char first[512], last[512];
  573. if (in->ver == AF_INET) {
  574. if (inet_ntop(AF_INET, in->first.addr8, first, INET_ADDRSTRLEN) &&
  575. inet_ntop(AF_INET, in->last.addr8, last, INET_ADDRSTRLEN))
  576. info("Adding values %s - %s to %s IP list \"%s\" used on network viewer",
  577. first, last,
  578. (*out == network_viewer_opt.included_ips)?"included":"excluded",
  579. table);
  580. } else {
  581. if (inet_ntop(AF_INET6, in->first.addr8, first, INET6_ADDRSTRLEN) &&
  582. inet_ntop(AF_INET6, in->last.addr8, last, INET6_ADDRSTRLEN))
  583. info("Adding values %s - %s to %s IP list \"%s\" used on network viewer",
  584. first, last,
  585. (*out == network_viewer_opt.included_ips)?"included":"excluded",
  586. table);
  587. }
  588. #endif
  589. }
  590. /**
  591. * Read Local Ports
  592. *
  593. * Parse /proc/net/{tcp,udp} and get the ports Linux is listening.
  594. *
  595. * @param filename the proc file to parse.
  596. * @param proto is the magic number associated to the protocol file we are reading.
  597. */
  598. static void read_local_ports(char *filename, uint8_t proto)
  599. {
  600. procfile *ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
  601. if (!ff)
  602. return;
  603. ff = procfile_readall(ff);
  604. if (!ff)
  605. return;
  606. size_t lines = procfile_lines(ff), l;
  607. for(l = 0; l < lines ;l++) {
  608. size_t words = procfile_linewords(ff, l);
  609. // This is header or end of file
  610. if (unlikely(words < 14))
  611. continue;
  612. // https://elixir.bootlin.com/linux/v5.7.8/source/include/net/tcp_states.h
  613. // 0A = TCP_LISTEN
  614. if (strcmp("0A", procfile_lineword(ff, l, 5)))
  615. continue;
  616. // Read local port
  617. uint16_t port = (uint16_t)strtol(procfile_lineword(ff, l, 2), NULL, 16);
  618. update_listen_table(htons(port), proto);
  619. }
  620. procfile_close(ff);
  621. }
  622. /**
  623. * Read Local addresseses
  624. *
  625. * Read the local address from the interfaces.
  626. */
  627. static void read_local_addresses()
  628. {
  629. struct ifaddrs *ifaddr, *ifa;
  630. if (getifaddrs(&ifaddr) == -1) {
  631. error("Cannot get the local IP addresses, it is no possible to do separation between inbound and outbound connections");
  632. return;
  633. }
  634. char *notext = { "No text representation" };
  635. for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) {
  636. if (ifa->ifa_addr == NULL)
  637. continue;
  638. if ((ifa->ifa_addr->sa_family != AF_INET) && (ifa->ifa_addr->sa_family != AF_INET6))
  639. continue;
  640. ebpf_network_viewer_ip_list_t *w = callocz(1, sizeof(ebpf_network_viewer_ip_list_t));
  641. int family = ifa->ifa_addr->sa_family;
  642. w->ver = (uint8_t) family;
  643. char text[INET6_ADDRSTRLEN];
  644. if (family == AF_INET) {
  645. struct sockaddr_in *in = (struct sockaddr_in*) ifa->ifa_addr;
  646. w->first.addr32[0] = in->sin_addr.s_addr;
  647. w->last.addr32[0] = in->sin_addr.s_addr;
  648. if (inet_ntop(AF_INET, w->first.addr8, text, INET_ADDRSTRLEN)) {
  649. w->value = strdupz(text);
  650. w->hash = simple_hash(text);
  651. } else {
  652. w->value = strdupz(notext);
  653. w->hash = simple_hash(notext);
  654. }
  655. } else {
  656. struct sockaddr_in6 *in6 = (struct sockaddr_in6*) ifa->ifa_addr;
  657. memcpy(w->first.addr8, (void *)&in6->sin6_addr, sizeof(struct in6_addr));
  658. memcpy(w->last.addr8, (void *)&in6->sin6_addr, sizeof(struct in6_addr));
  659. if (inet_ntop(AF_INET6, w->first.addr8, text, INET_ADDRSTRLEN)) {
  660. w->value = strdupz(text);
  661. w->hash = simple_hash(text);
  662. } else {
  663. w->value = strdupz(notext);
  664. w->hash = simple_hash(notext);
  665. }
  666. }
  667. fill_ip_list((family == AF_INET)?&network_viewer_opt.ipv4_local_ip:&network_viewer_opt.ipv6_local_ip,
  668. w,
  669. "selector");
  670. }
  671. freeifaddrs(ifaddr);
  672. }
  673. /**
  674. * Start Ptherad Variable
  675. *
  676. * This function starts all pthread variables.
  677. *
  678. * @return It returns 0 on success and -1.
  679. */
  680. int ebpf_start_pthread_variables()
  681. {
  682. pthread_mutex_init(&lock, NULL);
  683. pthread_mutex_init(&collect_data_mutex, NULL);
  684. if (pthread_cond_init(&collect_data_cond_var, NULL)) {
  685. thread_finished++;
  686. error("Cannot start conditional variable to control Apps charts.");
  687. return -1;
  688. }
  689. return 0;
  690. }
  691. /**
  692. * Allocate the vectors used for all threads.
  693. */
  694. static void ebpf_allocate_common_vectors()
  695. {
  696. all_pids = callocz((size_t)pid_max, sizeof(struct pid_stat *));
  697. global_process_stat = callocz((size_t)ebpf_nprocs, sizeof(ebpf_process_stat_t));
  698. }
  699. /**
  700. * Fill the ebpf_data structure with default values
  701. *
  702. * @param ef the pointer to set default values
  703. */
  704. void fill_ebpf_data(ebpf_data_t *ef)
  705. {
  706. memset(ef, 0, sizeof(ebpf_data_t));
  707. ef->kernel_string = kernel_string;
  708. ef->running_on_kernel = running_on_kernel;
  709. ef->map_fd = callocz(EBPF_MAX_MAPS, sizeof(int));
  710. ef->isrh = isrh;
  711. }
  712. /**
  713. * Define how to load the ebpf programs
  714. *
  715. * @param ptr the option given by users
  716. */
  717. static inline void how_to_load(char *ptr)
  718. {
  719. if (!strcasecmp(ptr, "return"))
  720. ebpf_set_thread_mode(MODE_RETURN);
  721. else if (!strcasecmp(ptr, "entry"))
  722. ebpf_set_thread_mode(MODE_ENTRY);
  723. else
  724. error("the option %s for \"ebpf load mode\" is not a valid option.", ptr);
  725. }
  726. /**
  727. * Fill Port list
  728. *
  729. * @param out a pointer to the link list.
  730. * @param in the structure that will be linked.
  731. */
  732. static inline void fill_port_list(ebpf_network_viewer_port_list_t **out, ebpf_network_viewer_port_list_t *in)
  733. {
  734. if (likely(*out)) {
  735. ebpf_network_viewer_port_list_t *move = *out, *store = *out;
  736. uint16_t first = ntohs(in->first);
  737. uint16_t last = ntohs(in->last);
  738. while (move) {
  739. uint16_t cmp_first = ntohs(move->first);
  740. uint16_t cmp_last = ntohs(move->last);
  741. if (cmp_first <= first && first <= cmp_last &&
  742. cmp_first <= last && last <= cmp_last ) {
  743. info("The range/value (%u, %u) is inside the range/value (%u, %u) already inserted, it will be ignored.",
  744. first, last, cmp_first, cmp_last);
  745. freez(in->value);
  746. freez(in);
  747. return;
  748. } else if (first <= cmp_first && cmp_first <= last &&
  749. first <= cmp_last && cmp_last <= last) {
  750. info("The range (%u, %u) is bigger than previous range (%u, %u) already inserted, the previous will be ignored.",
  751. first, last, cmp_first, cmp_last);
  752. freez(move->value);
  753. move->value = in->value;
  754. move->first = in->first;
  755. move->last = in->last;
  756. freez(in);
  757. return;
  758. }
  759. store = move;
  760. move = move->next;
  761. }
  762. store->next = in;
  763. } else {
  764. *out = in;
  765. }
  766. #ifdef NETDATA_INTERNAL_CHECKS
  767. info("Adding values %s( %u, %u) to %s port list used on network viewer",
  768. in->value, ntohs(in->first), ntohs(in->last),
  769. (*out == network_viewer_opt.included_port)?"included":"excluded");
  770. #endif
  771. }
  772. /**
  773. * Fill port list
  774. *
  775. * Fill an allocated port list with the range given
  776. *
  777. * @param out a pointer to store the link list
  778. * @param range the informed range for the user.
  779. */
  780. static void parse_port_list(void **out, char *range)
  781. {
  782. int first, last;
  783. ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out;
  784. char *copied = strdupz(range);
  785. if (*range == '*' && *(range+1) == '\0') {
  786. first = 1;
  787. last = 65535;
  788. clean_port_structure(list);
  789. goto fillenvpl;
  790. }
  791. char *end = range;
  792. //Move while I cannot find a separator
  793. while (*end && *end != ':' && *end != '-') end++;
  794. //It has a range
  795. if (likely(*end)) {
  796. *end++ = '\0';
  797. if (*end == '!') {
  798. info("The exclusion cannot be in the second part of the range, the range %s will be ignored.", copied);
  799. freez(copied);
  800. return;
  801. }
  802. last = str2i((const char *)end);
  803. } else {
  804. last = 0;
  805. }
  806. first = str2i((const char *)range);
  807. if (first < NETDATA_MINIMUM_PORT_VALUE || first > NETDATA_MAXIMUM_PORT_VALUE) {
  808. info("The first port %d of the range \"%s\" is invalid and it will be ignored!", first, copied);
  809. freez(copied);
  810. return;
  811. }
  812. if (!last)
  813. last = first;
  814. if (last < NETDATA_MINIMUM_PORT_VALUE || last > NETDATA_MAXIMUM_PORT_VALUE) {
  815. info("The second port %d of the range \"%s\" is invalid and the whole range will be ignored!", last, copied);
  816. freez(copied);
  817. return;
  818. }
  819. if (first > last) {
  820. info("The specified order %s is wrong, the smallest value is always the first, it will be ignored!", copied);
  821. freez(copied);
  822. return;
  823. }
  824. ebpf_network_viewer_port_list_t *w;
  825. fillenvpl:
  826. w = callocz(1, sizeof(ebpf_network_viewer_port_list_t));
  827. w->value = copied;
  828. w->hash = simple_hash(copied);
  829. w->first = (uint16_t)htons((uint16_t)first);
  830. w->last = (uint16_t)htons((uint16_t)last);
  831. w->cmp_first = (uint16_t)first;
  832. w->cmp_last = (uint16_t)last;
  833. fill_port_list(list, w);
  834. }
  835. /**
  836. * Parse Service List
  837. *
  838. * @param out a pointer to store the link list
  839. * @param service the service used to create the structure that will be linked.
  840. */
  841. static void parse_service_list(void **out, char *service)
  842. {
  843. ebpf_network_viewer_port_list_t **list = (ebpf_network_viewer_port_list_t **)out;
  844. struct servent *serv = getservbyname((const char *)service, "tcp");
  845. if (!serv)
  846. serv = getservbyname((const char *)service, "udp");
  847. if (!serv) {
  848. info("Cannot resolv the service '%s' with protocols TCP and UDP, it will be ignored", service);
  849. return;
  850. }
  851. ebpf_network_viewer_port_list_t *w = callocz(1, sizeof(ebpf_network_viewer_port_list_t));
  852. w->value = strdupz(service);
  853. w->hash = simple_hash(service);
  854. w->first = w->last = (uint16_t)serv->s_port;
  855. fill_port_list(list, w);
  856. }
  857. /**
  858. * Netmask
  859. *
  860. * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h)
  861. *
  862. * @param prefix create the netmask based in the CIDR value.
  863. *
  864. * @return
  865. */
  866. static inline in_addr_t netmask(int prefix) {
  867. if (prefix == 0)
  868. return (~((in_addr_t) - 1));
  869. else
  870. return (in_addr_t)(~((1 << (32 - prefix)) - 1));
  871. }
  872. /**
  873. * Broadcast
  874. *
  875. * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h)
  876. *
  877. * @param addr is the ip address
  878. * @param prefix is the CIDR value.
  879. *
  880. * @return It returns the last address of the range
  881. */
  882. static inline in_addr_t broadcast(in_addr_t addr, int prefix)
  883. {
  884. return (addr | ~netmask(prefix));
  885. }
  886. /**
  887. * Network
  888. *
  889. * Copied from iprange (https://github.com/firehol/iprange/blob/master/iprange.h)
  890. *
  891. * @param addr is the ip address
  892. * @param prefix is the CIDR value.
  893. *
  894. * @return It returns the first address of the range.
  895. */
  896. static inline in_addr_t ipv4_network(in_addr_t addr, int prefix)
  897. {
  898. return (addr & netmask(prefix));
  899. }
  900. /**
  901. * IP to network long
  902. *
  903. * @param dst the vector to store the result
  904. * @param ip the source ip given by our users.
  905. * @param domain the ip domain (IPV4 or IPV6)
  906. * @param source the original string
  907. *
  908. * @return it returns 0 on success and -1 otherwise.
  909. */
  910. static inline int ip2nl(uint8_t *dst, char *ip, int domain, char *source)
  911. {
  912. if (inet_pton(domain, ip, dst) <= 0) {
  913. error("The address specified (%s) is invalid ", source);
  914. return -1;
  915. }
  916. return 0;
  917. }
  918. /**
  919. * Get IPV6 Last Address
  920. *
  921. * @param out the address to store the last address.
  922. * @param in the address used to do the math.
  923. * @param prefix number of bits used to calculate the address
  924. */
  925. static void get_ipv6_last_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix)
  926. {
  927. uint64_t mask,tmp;
  928. uint64_t ret[2];
  929. memcpy(ret, in->addr32, sizeof(union netdata_ip_t));
  930. if (prefix == 128) {
  931. memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t));
  932. return;
  933. } else if (!prefix) {
  934. ret[0] = ret[1] = 0xFFFFFFFFFFFFFFFF;
  935. memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
  936. return;
  937. } else if (prefix <= 64) {
  938. ret[1] = 0xFFFFFFFFFFFFFFFFULL;
  939. tmp = be64toh(ret[0]);
  940. if (prefix > 0) {
  941. mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix);
  942. tmp |= ~mask;
  943. }
  944. ret[0] = htobe64(tmp);
  945. } else {
  946. mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix);
  947. tmp = be64toh(ret[1]);
  948. tmp |= ~mask;
  949. ret[1] = htobe64(tmp);
  950. }
  951. memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
  952. }
  953. /**
  954. * Calculate ipv6 first address
  955. *
  956. * @param out the address to store the first address.
  957. * @param in the address used to do the math.
  958. * @param prefix number of bits used to calculate the address
  959. */
  960. static void get_ipv6_first_addr(union netdata_ip_t *out, union netdata_ip_t *in, uint64_t prefix)
  961. {
  962. uint64_t mask,tmp;
  963. uint64_t ret[2];
  964. memcpy(ret, in->addr32, sizeof(union netdata_ip_t));
  965. if (prefix == 128) {
  966. memcpy(out->addr32, in->addr32, sizeof(union netdata_ip_t));
  967. return;
  968. } else if (!prefix) {
  969. ret[0] = ret[1] = 0;
  970. memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
  971. return;
  972. } else if (prefix <= 64) {
  973. ret[1] = 0ULL;
  974. tmp = be64toh(ret[0]);
  975. if (prefix > 0) {
  976. mask = 0xFFFFFFFFFFFFFFFFULL << (64 - prefix);
  977. tmp &= mask;
  978. }
  979. ret[0] = htobe64(tmp);
  980. } else {
  981. mask = 0xFFFFFFFFFFFFFFFFULL << (128 - prefix);
  982. tmp = be64toh(ret[1]);
  983. tmp &= mask;
  984. ret[1] = htobe64(tmp);
  985. }
  986. memcpy(out->addr32, ret, sizeof(union netdata_ip_t));
  987. }
  988. /**
  989. * Parse IP List
  990. *
  991. * Parse IP list and link it.
  992. *
  993. * @param out a pointer to store the link list
  994. * @param ip the value given as parameter
  995. */
  996. static void parse_ip_list(void **out, char *ip)
  997. {
  998. ebpf_network_viewer_ip_list_t **list = (ebpf_network_viewer_ip_list_t **)out;
  999. char *ipdup = strdupz(ip);
  1000. union netdata_ip_t first = { };
  1001. union netdata_ip_t last = { };
  1002. char *is_ipv6;
  1003. if (*ip == '*' && *(ip+1) == '\0') {
  1004. memset(first.addr8, 0, sizeof(first.addr8));
  1005. memset(last.addr8, 0xFF, sizeof(last.addr8));
  1006. is_ipv6 = ip;
  1007. clean_ip_structure(list);
  1008. goto storethisip;
  1009. }
  1010. char *end = ip;
  1011. // Move while I cannot find a separator
  1012. while (*end && *end != '/' && *end != '-') end++;
  1013. // We will use only the classic IPV6 for while, but we could consider the base 85 in a near future
  1014. // https://tools.ietf.org/html/rfc1924
  1015. is_ipv6 = strchr(ip, ':');
  1016. int select;
  1017. if (*end && !is_ipv6) { // IPV4 range
  1018. select = (*end == '/') ? 0 : 1;
  1019. *end++ = '\0';
  1020. if (*end == '!') {
  1021. info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup);
  1022. goto cleanipdup;
  1023. }
  1024. if (!select) { // CIDR
  1025. select = ip2nl(first.addr8, ip, AF_INET, ipdup);
  1026. if (select)
  1027. goto cleanipdup;
  1028. select = (int) str2i(end);
  1029. if (select < NETDATA_MINIMUM_IPV4_CIDR || select > NETDATA_MAXIMUM_IPV4_CIDR) {
  1030. info("The specified CIDR %s is not valid, the IP %s will be ignored.", end, ip);
  1031. goto cleanipdup;
  1032. }
  1033. last.addr32[0] = htonl(broadcast(ntohl(first.addr32[0]), select));
  1034. // This was added to remove
  1035. // https://app.codacy.com/manual/netdata/netdata/pullRequest?prid=5810941&bid=19021977
  1036. UNUSED(last.addr32[0]);
  1037. uint32_t ipv4_test = htonl(ipv4_network(ntohl(first.addr32[0]), select));
  1038. if (first.addr32[0] != ipv4_test) {
  1039. first.addr32[0] = ipv4_test;
  1040. struct in_addr ipv4_convert;
  1041. ipv4_convert.s_addr = ipv4_test;
  1042. char ipv4_msg[INET_ADDRSTRLEN];
  1043. if(inet_ntop(AF_INET, &ipv4_convert, ipv4_msg, INET_ADDRSTRLEN))
  1044. info("The network value of CIDR %s was updated for %s .", ipdup, ipv4_msg);
  1045. }
  1046. } else { // Range
  1047. select = ip2nl(first.addr8, ip, AF_INET, ipdup);
  1048. if (select)
  1049. goto cleanipdup;
  1050. select = ip2nl(last.addr8, end, AF_INET, ipdup);
  1051. if (select)
  1052. goto cleanipdup;
  1053. }
  1054. if (htonl(first.addr32[0]) > htonl(last.addr32[0])) {
  1055. info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.",
  1056. ipdup);
  1057. goto cleanipdup;
  1058. }
  1059. } else if (is_ipv6) { // IPV6
  1060. if (!*end) { // Unique
  1061. select = ip2nl(first.addr8, ip, AF_INET6, ipdup);
  1062. if (select)
  1063. goto cleanipdup;
  1064. memcpy(last.addr8, first.addr8, sizeof(first.addr8));
  1065. } else if (*end == '-') {
  1066. *end++ = 0x00;
  1067. if (*end == '!') {
  1068. info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup);
  1069. goto cleanipdup;
  1070. }
  1071. select = ip2nl(first.addr8, ip, AF_INET6, ipdup);
  1072. if (select)
  1073. goto cleanipdup;
  1074. select = ip2nl(last.addr8, end, AF_INET6, ipdup);
  1075. if (select)
  1076. goto cleanipdup;
  1077. } else { // CIDR
  1078. *end++ = 0x00;
  1079. if (*end == '!') {
  1080. info("The exclusion cannot be in the second part of the range %s, it will be ignored.", ipdup);
  1081. goto cleanipdup;
  1082. }
  1083. select = str2i(end);
  1084. if (select < 0 || select > 128) {
  1085. info("The CIDR %s is not valid, the address %s will be ignored.", end, ip);
  1086. goto cleanipdup;
  1087. }
  1088. uint64_t prefix = (uint64_t)select;
  1089. select = ip2nl(first.addr8, ip, AF_INET6, ipdup);
  1090. if (select)
  1091. goto cleanipdup;
  1092. get_ipv6_last_addr(&last, &first, prefix);
  1093. union netdata_ip_t ipv6_test;
  1094. get_ipv6_first_addr(&ipv6_test, &first, prefix);
  1095. if (memcmp(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t)) != 0) {
  1096. memcpy(first.addr8, ipv6_test.addr8, sizeof(union netdata_ip_t));
  1097. struct in6_addr ipv6_convert;
  1098. memcpy(ipv6_convert.s6_addr, ipv6_test.addr8, sizeof(union netdata_ip_t));
  1099. char ipv6_msg[INET6_ADDRSTRLEN];
  1100. if(inet_ntop(AF_INET6, &ipv6_convert, ipv6_msg, INET6_ADDRSTRLEN))
  1101. info("The network value of CIDR %s was updated for %s .", ipdup, ipv6_msg);
  1102. }
  1103. }
  1104. if ((be64toh(*(uint64_t *)&first.addr32[2]) > be64toh(*(uint64_t *)&last.addr32[2]) &&
  1105. !memcmp(first.addr32, last.addr32, 2*sizeof(uint32_t))) ||
  1106. (be64toh(*(uint64_t *)&first.addr32) > be64toh(*(uint64_t *)&last.addr32)) ) {
  1107. info("The specified range %s is invalid, the second address is smallest than the first, it will be ignored.",
  1108. ipdup);
  1109. goto cleanipdup;
  1110. }
  1111. } else { // Unique ip
  1112. select = ip2nl(first.addr8, ip, AF_INET, ipdup);
  1113. if (select)
  1114. goto cleanipdup;
  1115. memcpy(last.addr8, first.addr8, sizeof(first.addr8));
  1116. }
  1117. ebpf_network_viewer_ip_list_t *store;
  1118. storethisip:
  1119. store = callocz(1, sizeof(ebpf_network_viewer_ip_list_t));
  1120. store->value = ipdup;
  1121. store->hash = simple_hash(ipdup);
  1122. store->ver = (uint8_t)(!is_ipv6)?AF_INET:AF_INET6;
  1123. memcpy(store->first.addr8, first.addr8, sizeof(first.addr8));
  1124. memcpy(store->last.addr8, last.addr8, sizeof(last.addr8));
  1125. fill_ip_list(list, store, "socket");
  1126. return;
  1127. cleanipdup:
  1128. freez(ipdup);
  1129. }
  1130. /**
  1131. * Parse IP Range
  1132. *
  1133. * Parse the IP ranges given and create Network Viewer IP Structure
  1134. *
  1135. * @param ptr is a pointer with the text to parse.
  1136. */
  1137. static void parse_ips(char *ptr)
  1138. {
  1139. // No value
  1140. if (unlikely(!ptr))
  1141. return;
  1142. while (likely(ptr)) {
  1143. // Move forward until next valid character
  1144. while (isspace(*ptr)) ptr++;
  1145. // No valid value found
  1146. if (unlikely(!*ptr))
  1147. return;
  1148. // Find space that ends the list
  1149. char *end = strchr(ptr, ' ');
  1150. if (end) {
  1151. *end++ = '\0';
  1152. }
  1153. int neg = 0;
  1154. if (*ptr == '!') {
  1155. neg++;
  1156. ptr++;
  1157. }
  1158. if (isascii(*ptr)) { // Parse port
  1159. parse_ip_list((!neg)?(void **)&network_viewer_opt.included_ips:(void **)&network_viewer_opt.excluded_ips,
  1160. ptr);
  1161. }
  1162. ptr = end;
  1163. }
  1164. }
  1165. /**
  1166. * Parse Port Range
  1167. *
  1168. * Parse the port ranges given and create Network Viewer Port Structure
  1169. *
  1170. * @param ptr is a pointer with the text to parse.
  1171. */
  1172. static void parse_ports(char *ptr)
  1173. {
  1174. // No value
  1175. if (unlikely(!ptr))
  1176. return;
  1177. while (likely(ptr)) {
  1178. // Move forward until next valid character
  1179. while (isspace(*ptr)) ptr++;
  1180. // No valid value found
  1181. if (unlikely(!*ptr))
  1182. return;
  1183. // Find space that ends the list
  1184. char *end = strchr(ptr, ' ');
  1185. if (end) {
  1186. *end++ = '\0';
  1187. }
  1188. int neg = 0;
  1189. if (*ptr == '!') {
  1190. neg++;
  1191. ptr++;
  1192. }
  1193. if (isdigit(*ptr)) { // Parse port
  1194. parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port,
  1195. ptr);
  1196. } else if (isalpha(*ptr)) { // Parse service
  1197. parse_service_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port,
  1198. ptr);
  1199. } else if (*ptr == '*') { // All
  1200. parse_port_list((!neg)?(void **)&network_viewer_opt.included_port:(void **)&network_viewer_opt.excluded_port,
  1201. ptr);
  1202. }
  1203. ptr = end;
  1204. }
  1205. }
  1206. /**
  1207. * Link hostname
  1208. *
  1209. * @param out is the output link list
  1210. * @param in the hostname to add to list.
  1211. */
  1212. static void link_hostname(ebpf_network_viewer_hostname_list_t **out, ebpf_network_viewer_hostname_list_t *in)
  1213. {
  1214. if (likely(*out)) {
  1215. ebpf_network_viewer_hostname_list_t *move = *out;
  1216. for (; move->next ; move = move->next ) {
  1217. if (move->hash == in->hash && !strcmp(move->value, in->value)) {
  1218. info("The hostname %s was already inserted, it will be ignored.", in->value);
  1219. freez(in->value);
  1220. simple_pattern_free(in->value_pattern);
  1221. freez(in);
  1222. return;
  1223. }
  1224. }
  1225. move->next = in;
  1226. } else {
  1227. *out = in;
  1228. }
  1229. #ifdef NETDATA_INTERNAL_CHECKS
  1230. info("Adding value %s to %s hostname list used on network viewer",
  1231. in->value,
  1232. (*out == network_viewer_opt.included_hostnames)?"included":"excluded");
  1233. #endif
  1234. }
  1235. /**
  1236. * Link Hostnames
  1237. *
  1238. * Parse the list of hostnames to create the link list.
  1239. * This is not associated with the IP, because simple patterns like *example* cannot be resolved to IP.
  1240. *
  1241. * @param out is the output link list
  1242. * @param parse is a pointer with the text to parser.
  1243. */
  1244. static void link_hostnames(char *parse)
  1245. {
  1246. // No value
  1247. if (unlikely(!parse))
  1248. return;
  1249. while (likely(parse)) {
  1250. // Find the first valid value
  1251. while (isspace(*parse)) parse++;
  1252. // No valid value found
  1253. if (unlikely(!*parse))
  1254. return;
  1255. // Find space that ends the list
  1256. char *end = strchr(parse, ' ');
  1257. if (end) {
  1258. *end++ = '\0';
  1259. }
  1260. int neg = 0;
  1261. if (*parse == '!') {
  1262. neg++;
  1263. parse++;
  1264. }
  1265. ebpf_network_viewer_hostname_list_t *hostname = callocz(1 , sizeof(ebpf_network_viewer_hostname_list_t));
  1266. hostname->value = strdupz(parse);
  1267. hostname->hash = simple_hash(parse);
  1268. hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT);
  1269. link_hostname((!neg)?&network_viewer_opt.included_hostnames:&network_viewer_opt.excluded_hostnames,
  1270. hostname);
  1271. parse = end;
  1272. }
  1273. }
  1274. /**
  1275. * Read max dimension.
  1276. *
  1277. * Netdata plot two dimensions per connection, so it is necessary to adjust the values.
  1278. */
  1279. static void read_max_dimension()
  1280. {
  1281. int maxdim ;
  1282. maxdim = (int) appconfig_get_number(&collector_config,
  1283. EBPF_NETWORK_VIEWER_SECTION,
  1284. "maximum dimensions",
  1285. NETDATA_NV_CAP_VALUE);
  1286. if (maxdim < 0) {
  1287. error("'maximum dimensions = %d' must be a positive number, Netdata will change for default value %ld.",
  1288. maxdim, NETDATA_NV_CAP_VALUE);
  1289. maxdim = NETDATA_NV_CAP_VALUE;
  1290. }
  1291. maxdim /= 2;
  1292. if (!maxdim) {
  1293. info("The number of dimensions is too small (%u), we are setting it to minimum 2", network_viewer_opt.max_dim);
  1294. network_viewer_opt.max_dim = 1;
  1295. }
  1296. network_viewer_opt.max_dim = (uint32_t)maxdim;
  1297. }
  1298. /**
  1299. * Parse network viewer section
  1300. */
  1301. static void parse_network_viewer_section()
  1302. {
  1303. read_max_dimension();
  1304. network_viewer_opt.hostname_resolution_enabled = appconfig_get_boolean(&collector_config,
  1305. EBPF_NETWORK_VIEWER_SECTION,
  1306. "resolve hostnames",
  1307. CONFIG_BOOLEAN_NO);
  1308. network_viewer_opt.service_resolution_enabled = appconfig_get_boolean(&collector_config,
  1309. EBPF_NETWORK_VIEWER_SECTION,
  1310. "resolve service names",
  1311. CONFIG_BOOLEAN_NO);
  1312. char *value = appconfig_get(&collector_config, EBPF_NETWORK_VIEWER_SECTION,
  1313. "ports", NULL);
  1314. parse_ports(value);
  1315. if (network_viewer_opt.hostname_resolution_enabled) {
  1316. value = appconfig_get(&collector_config, EBPF_NETWORK_VIEWER_SECTION, "hostnames", NULL);
  1317. link_hostnames(value);
  1318. } else {
  1319. info("Name resolution is disabled, collector will not parser \"hostnames\" list.");
  1320. }
  1321. value = appconfig_get(&collector_config, EBPF_NETWORK_VIEWER_SECTION,
  1322. "ips", "!127.0.0.1/8 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 fc00::/7 !::1/128");
  1323. parse_ips(value);
  1324. }
  1325. /**
  1326. * Link dimension name
  1327. *
  1328. * Link user specified names inside a link list.
  1329. *
  1330. * @param port the port number associated to the dimension name.
  1331. * @param hash the calculated hash for the dimension name.
  1332. * @param name the dimension name.
  1333. */
  1334. static void link_dimension_name(char *port, uint32_t hash, char *value)
  1335. {
  1336. int test = str2i(port);
  1337. if (test < NETDATA_MINIMUM_PORT_VALUE || test > NETDATA_MAXIMUM_PORT_VALUE){
  1338. error("The dimension given (%s = %s) has an invalid value and it will be ignored.", port, value);
  1339. return;
  1340. }
  1341. ebpf_network_viewer_dim_name_t *w;
  1342. w = callocz(1, sizeof(ebpf_network_viewer_dim_name_t));
  1343. w->name = strdupz(value);
  1344. w->hash = hash;
  1345. w->port = (uint16_t) htons(test);
  1346. ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names;
  1347. if (unlikely(!names)) {
  1348. network_viewer_opt.names = w;
  1349. } else {
  1350. for (; names->next; names = names->next) {
  1351. if (names->port == w->port) {
  1352. info("Dupplicated definition for a service, the name %s will be ignored. ", names->name);
  1353. freez(names->name);
  1354. names->name = w->name;
  1355. names->hash = w->hash;
  1356. freez(w);
  1357. return;
  1358. }
  1359. }
  1360. names->next = w;
  1361. }
  1362. #ifdef NETDATA_INTERNAL_CHECKS
  1363. info("Adding values %s( %u) to dimension name list used on network viewer", w->name, htons(w->port));
  1364. #endif
  1365. }
  1366. /**
  1367. * Parse service Name section.
  1368. *
  1369. * This function gets the values that will be used to overwrite dimensions.
  1370. */
  1371. static void parse_service_name_section()
  1372. {
  1373. struct section *co = appconfig_get_section(&collector_config, EBPF_SERVICE_NAME_SECTION);
  1374. if (co) {
  1375. struct config_option *cv;
  1376. for (cv = co->values; cv ; cv = cv->next) {
  1377. link_dimension_name(cv->name, cv->hash, cv->value);
  1378. }
  1379. }
  1380. // Always associated the default port to Netdata
  1381. ebpf_network_viewer_dim_name_t *names = network_viewer_opt.names;
  1382. if (names) {
  1383. uint16_t default_port = htons(19999);
  1384. while (names) {
  1385. if (names->port == default_port)
  1386. return;
  1387. names = names->next;
  1388. }
  1389. }
  1390. char *port_string = getenv("NETDATA_LISTEN_PORT");
  1391. if (port_string)
  1392. link_dimension_name(port_string, simple_hash(port_string), "Netdata");
  1393. }
  1394. /**
  1395. * Read collector values
  1396. *
  1397. * @param disable_apps variable to store information related to apps.
  1398. */
  1399. static void read_collector_values(int *disable_apps)
  1400. {
  1401. // Read global section
  1402. char *value;
  1403. if (appconfig_exists(&collector_config, EBPF_GLOBAL_SECTION, "load")) // Backward compatibility
  1404. value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, "load", "entry");
  1405. else
  1406. value = appconfig_get(&collector_config, EBPF_GLOBAL_SECTION, "ebpf load mode", "entry");
  1407. how_to_load(value);
  1408. // This is kept to keep compatibility
  1409. uint32_t enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, "disable apps",
  1410. CONFIG_BOOLEAN_NO);
  1411. if (!enabled) {
  1412. // Apps is a positive sentence, so we need to invert the values to disable apps.
  1413. enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, "apps",
  1414. CONFIG_BOOLEAN_YES);
  1415. enabled = (enabled == CONFIG_BOOLEAN_NO)?CONFIG_BOOLEAN_YES:CONFIG_BOOLEAN_NO;
  1416. }
  1417. *disable_apps = (int)enabled;
  1418. // Read ebpf programs section
  1419. enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION,
  1420. ebpf_modules[0].config_name, CONFIG_BOOLEAN_YES);
  1421. int started = 0;
  1422. if (enabled) {
  1423. ebpf_enable_chart(EBPF_MODULE_PROCESS_IDX, *disable_apps);
  1424. started++;
  1425. }
  1426. // This is kept to keep compatibility
  1427. enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network viewer",
  1428. CONFIG_BOOLEAN_NO);
  1429. if (!enabled)
  1430. enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, ebpf_modules[1].config_name,
  1431. CONFIG_BOOLEAN_NO);
  1432. if (enabled) {
  1433. ebpf_enable_chart(EBPF_MODULE_SOCKET_IDX, *disable_apps);
  1434. // Read network viewer section if network viewer is enabled
  1435. parse_network_viewer_section();
  1436. parse_service_name_section();
  1437. started++;
  1438. }
  1439. // This is kept to keep compatibility
  1440. enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network connection monitoring",
  1441. CONFIG_BOOLEAN_NO);
  1442. if (!enabled)
  1443. enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "network connections",
  1444. CONFIG_BOOLEAN_NO);
  1445. ebpf_modules[1].optional = enabled;
  1446. if (!started){
  1447. ebpf_enable_all_charts(*disable_apps);
  1448. // Read network viewer section
  1449. parse_network_viewer_section();
  1450. parse_service_name_section();
  1451. }
  1452. }
  1453. /**
  1454. * Load collector config
  1455. *
  1456. * @param path the path where the file ebpf.conf is stored.
  1457. * @param disable_apps variable to store the information about apps plugin status.
  1458. *
  1459. * @return 0 on success and -1 otherwise.
  1460. */
  1461. static int load_collector_config(char *path, int *disable_apps)
  1462. {
  1463. char lpath[4096];
  1464. snprintf(lpath, 4095, "%s/%s", path, "ebpf.conf");
  1465. if (!appconfig_load(&collector_config, lpath, 0, NULL))
  1466. return -1;
  1467. read_collector_values(disable_apps);
  1468. return 0;
  1469. }
  1470. /**
  1471. * Set global variables reading environment variables
  1472. */
  1473. void set_global_variables()
  1474. {
  1475. // Get environment variables
  1476. ebpf_plugin_dir = getenv("NETDATA_PLUGINS_DIR");
  1477. if (!ebpf_plugin_dir)
  1478. ebpf_plugin_dir = PLUGINS_DIR;
  1479. ebpf_user_config_dir = getenv("NETDATA_USER_CONFIG_DIR");
  1480. if (!ebpf_user_config_dir)
  1481. ebpf_user_config_dir = CONFIG_DIR;
  1482. ebpf_stock_config_dir = getenv("NETDATA_STOCK_CONFIG_DIR");
  1483. if (!ebpf_stock_config_dir)
  1484. ebpf_stock_config_dir = LIBCONFIG_DIR;
  1485. ebpf_configured_log_dir = getenv("NETDATA_LOG_DIR");
  1486. if (!ebpf_configured_log_dir)
  1487. ebpf_configured_log_dir = LOG_DIR;
  1488. ebpf_nprocs = (int)sysconf(_SC_NPROCESSORS_ONLN);
  1489. if (ebpf_nprocs > NETDATA_MAX_PROCESSOR) {
  1490. ebpf_nprocs = NETDATA_MAX_PROCESSOR;
  1491. }
  1492. isrh = get_redhat_release();
  1493. pid_max = get_system_pid_max();
  1494. }
  1495. /**
  1496. * Parse arguments given from user.
  1497. *
  1498. * @param argc the number of arguments
  1499. * @param argv the pointer to the arguments
  1500. */
  1501. static void parse_args(int argc, char **argv)
  1502. {
  1503. int enabled = 0;
  1504. int disable_apps = 0;
  1505. int freq = 0;
  1506. int option_index = 0;
  1507. static struct option long_options[] = {
  1508. {"help", no_argument, 0, 'h' },
  1509. {"version", no_argument, 0, 'v' },
  1510. {"global", no_argument, 0, 'g' },
  1511. {"all", no_argument, 0, 'a' },
  1512. {"net", no_argument, 0, 'n' },
  1513. {"process", no_argument, 0, 'p' },
  1514. {"return", no_argument, 0, 'r' },
  1515. {0, 0, 0, 0}
  1516. };
  1517. memset(&network_viewer_opt, 0, sizeof(network_viewer_opt));
  1518. network_viewer_opt.max_dim = NETDATA_NV_CAP_VALUE;
  1519. if (argc > 1) {
  1520. int n = (int)str2l(argv[1]);
  1521. if (n > 0) {
  1522. freq = n;
  1523. }
  1524. }
  1525. while (1) {
  1526. int c = getopt_long(argc, argv, "hvganpr", long_options, &option_index);
  1527. if (c == -1)
  1528. break;
  1529. switch (c) {
  1530. case 'h': {
  1531. ebpf_print_help();
  1532. exit(0);
  1533. }
  1534. case 'v': {
  1535. printf("ebpf.plugin %s\n", VERSION);
  1536. exit(0);
  1537. }
  1538. case 'g': {
  1539. disable_apps = 1;
  1540. ebpf_disable_apps();
  1541. #ifdef NETDATA_INTERNAL_CHECKS
  1542. info(
  1543. "EBPF running with global chart group, because it was started with the option \"--global\" or \"-g\".");
  1544. #endif
  1545. break;
  1546. }
  1547. case 'a': {
  1548. ebpf_enable_all_charts(disable_apps);
  1549. #ifdef NETDATA_INTERNAL_CHECKS
  1550. info("EBPF running with all chart groups, because it was started with the option \"--all\" or \"-a\".");
  1551. #endif
  1552. break;
  1553. }
  1554. case 'n': {
  1555. enabled = 1;
  1556. ebpf_enable_chart(EBPF_MODULE_SOCKET_IDX, disable_apps);
  1557. #ifdef NETDATA_INTERNAL_CHECKS
  1558. info("EBPF enabling \"NET\" charts, because it was started with the option \"--net\" or \"-n\".");
  1559. #endif
  1560. break;
  1561. }
  1562. case 'p': {
  1563. enabled = 1;
  1564. ebpf_enable_chart(EBPF_MODULE_PROCESS_IDX, disable_apps);
  1565. #ifdef NETDATA_INTERNAL_CHECKS
  1566. info(
  1567. "EBPF enabling \"PROCESS\" charts, because it was started with the option \"--process\" or \"-p\".");
  1568. #endif
  1569. break;
  1570. }
  1571. case 'r': {
  1572. ebpf_set_thread_mode(MODE_RETURN);
  1573. #ifdef NETDATA_INTERNAL_CHECKS
  1574. info("EBPF running in \"return\" mode, because it was started with the option \"--return\" or \"-r\".");
  1575. #endif
  1576. break;
  1577. }
  1578. default: {
  1579. break;
  1580. }
  1581. }
  1582. }
  1583. if (freq > 0) {
  1584. update_every = freq;
  1585. }
  1586. if (load_collector_config(ebpf_user_config_dir, &disable_apps)) {
  1587. info(
  1588. "Does not have a configuration file inside `%s/ebpf.conf. It will try to load stock file.",
  1589. ebpf_user_config_dir);
  1590. if (load_collector_config(ebpf_stock_config_dir, &disable_apps)) {
  1591. info("Does not have a stock file. It is starting with default options.");
  1592. } else {
  1593. enabled = 1;
  1594. }
  1595. } else {
  1596. enabled = 1;
  1597. }
  1598. if (!enabled) {
  1599. ebpf_enable_all_charts(disable_apps);
  1600. #ifdef NETDATA_INTERNAL_CHECKS
  1601. info("EBPF running with all charts, because neither \"-n\" or \"-p\" was given.");
  1602. #endif
  1603. }
  1604. if (disable_apps)
  1605. return;
  1606. // Load apps_groups.conf
  1607. if (ebpf_read_apps_groups_conf(
  1608. &apps_groups_default_target, &apps_groups_root_target, ebpf_user_config_dir, "groups")) {
  1609. info(
  1610. "Cannot read process groups configuration file '%s/apps_groups.conf'. Will try '%s/apps_groups.conf'",
  1611. ebpf_user_config_dir, ebpf_stock_config_dir);
  1612. if (ebpf_read_apps_groups_conf(
  1613. &apps_groups_default_target, &apps_groups_root_target, ebpf_stock_config_dir, "groups")) {
  1614. error(
  1615. "Cannot read process groups '%s/apps_groups.conf'. There are no internal defaults. Failing.",
  1616. ebpf_stock_config_dir);
  1617. thread_finished++;
  1618. ebpf_exit(1);
  1619. }
  1620. } else
  1621. info("Loaded config file '%s/apps_groups.conf'", ebpf_user_config_dir);
  1622. }
  1623. /*****************************************************************
  1624. *
  1625. * COLLECTOR ENTRY POINT
  1626. *
  1627. *****************************************************************/
  1628. /**
  1629. * Entry point
  1630. *
  1631. * @param argc the number of arguments
  1632. * @param argv the pointer to the arguments
  1633. *
  1634. * @return it returns 0 on success and another integer otherwise
  1635. */
  1636. int main(int argc, char **argv)
  1637. {
  1638. set_global_variables();
  1639. parse_args(argc, argv);
  1640. running_on_kernel = get_kernel_version(kernel_string, 63);
  1641. if (!has_condition_to_run(running_on_kernel)) {
  1642. error("The current collector cannot run on this kernel.");
  1643. return 2;
  1644. }
  1645. if (!am_i_running_as_root()) {
  1646. error(
  1647. "ebpf.plugin should either run as root (now running with uid %u, euid %u) or have special capabilities..",
  1648. (unsigned int)getuid(), (unsigned int)geteuid());
  1649. return 3;
  1650. }
  1651. // set name
  1652. program_name = "ebpf.plugin";
  1653. // disable syslog
  1654. error_log_syslog = 0;
  1655. // set errors flood protection to 100 logs per hour
  1656. error_log_errors_per_period = 100;
  1657. error_log_throttle_period = 3600;
  1658. struct rlimit r = { RLIM_INFINITY, RLIM_INFINITY };
  1659. if (setrlimit(RLIMIT_MEMLOCK, &r)) {
  1660. error("Setrlimit(RLIMIT_MEMLOCK)");
  1661. return 4;
  1662. }
  1663. signal(SIGINT, ebpf_exit);
  1664. signal(SIGTERM, ebpf_exit);
  1665. signal(SIGPIPE, ebpf_exit);
  1666. if (ebpf_start_pthread_variables()) {
  1667. thread_finished++;
  1668. error("Cannot start mutex to control overall charts.");
  1669. ebpf_exit(5);
  1670. }
  1671. ebpf_allocate_common_vectors();
  1672. read_local_addresses();
  1673. read_local_ports("/proc/net/tcp", IPPROTO_TCP);
  1674. read_local_ports("/proc/net/tcp6", IPPROTO_TCP);
  1675. read_local_ports("/proc/net/udp", IPPROTO_UDP);
  1676. read_local_ports("/proc/net/udp6", IPPROTO_UDP);
  1677. struct netdata_static_thread ebpf_threads[] = {
  1678. {"EBPF PROCESS", NULL, NULL, 1, NULL, NULL, ebpf_modules[0].start_routine},
  1679. {"EBPF SOCKET" , NULL, NULL, 1, NULL, NULL, ebpf_modules[1].start_routine},
  1680. {NULL , NULL, NULL, 0, NULL, NULL, NULL}
  1681. };
  1682. //clean_loaded_events();
  1683. int i;
  1684. for (i = 0; ebpf_threads[i].name != NULL; i++) {
  1685. struct netdata_static_thread *st = &ebpf_threads[i];
  1686. st->thread = mallocz(sizeof(netdata_thread_t));
  1687. ebpf_module_t *em = &ebpf_modules[i];
  1688. em->thread_id = i;
  1689. netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_JOINABLE, st->start_routine, em);
  1690. }
  1691. for (i = 0; ebpf_threads[i].name != NULL; i++) {
  1692. struct netdata_static_thread *st = &ebpf_threads[i];
  1693. netdata_thread_join(*st->thread, NULL);
  1694. }
  1695. thread_finished++;
  1696. ebpf_exit(0);
  1697. return 0;
  1698. }