ebpf_cachestat.c 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336
  1. // SPDX-License-Identifier: GPL-3.0-or-later
  2. #include "ebpf.h"
  3. #include "ebpf_cachestat.h"
  4. static char *cachestat_counter_dimension_name[NETDATA_CACHESTAT_END] = { "ratio", "dirty", "hit",
  5. "miss" };
  6. static netdata_syscall_stat_t cachestat_counter_aggregated_data[NETDATA_CACHESTAT_END];
  7. static netdata_publish_syscall_t cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_END];
  8. netdata_cachestat_pid_t *cachestat_vector = NULL;
  9. static netdata_idx_t cachestat_hash_values[NETDATA_CACHESTAT_END];
  10. static netdata_idx_t *cachestat_values = NULL;
  11. ebpf_local_maps_t cachestat_maps[] = {{.name = "cstat_global", .internal_input = NETDATA_CACHESTAT_END,
  12. .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
  13. .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED,
  14. #ifdef LIBBPF_MAJOR_VERSION
  15. .map_type = BPF_MAP_TYPE_PERCPU_ARRAY
  16. #endif
  17. },
  18. {.name = "cstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
  19. .user_input = 0,
  20. .type = NETDATA_EBPF_MAP_RESIZABLE | NETDATA_EBPF_MAP_PID,
  21. .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED,
  22. #ifdef LIBBPF_MAJOR_VERSION
  23. .map_type = BPF_MAP_TYPE_PERCPU_HASH
  24. #endif
  25. },
  26. {.name = "cstat_ctrl", .internal_input = NETDATA_CONTROLLER_END,
  27. .user_input = 0,
  28. .type = NETDATA_EBPF_MAP_CONTROLLER,
  29. .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED,
  30. #ifdef LIBBPF_MAJOR_VERSION
  31. .map_type = BPF_MAP_TYPE_PERCPU_ARRAY
  32. #endif
  33. },
  34. {.name = NULL, .internal_input = 0, .user_input = 0,
  35. .type = NETDATA_EBPF_MAP_CONTROLLER,
  36. .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED,
  37. #ifdef LIBBPF_MAJOR_VERSION
  38. #endif
  39. }};
  40. struct config cachestat_config = { .first_section = NULL,
  41. .last_section = NULL,
  42. .mutex = NETDATA_MUTEX_INITIALIZER,
  43. .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
  44. .rwlock = AVL_LOCK_INITIALIZER } };
  45. netdata_ebpf_targets_t cachestat_targets[] = { {.name = "add_to_page_cache_lru", .mode = EBPF_LOAD_TRAMPOLINE},
  46. {.name = "mark_page_accessed", .mode = EBPF_LOAD_TRAMPOLINE},
  47. {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE},
  48. {.name = "mark_buffer_dirty", .mode = EBPF_LOAD_TRAMPOLINE},
  49. {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
  50. static char *account_page[NETDATA_CACHESTAT_ACCOUNT_DIRTY_END] ={ "account_page_dirtied",
  51. "__set_page_dirty", "__folio_mark_dirty" };
  52. #ifdef LIBBPF_MAJOR_VERSION
  53. /**
  54. * Disable probe
  55. *
  56. * Disable all probes to use exclusively another method.
  57. *
  58. * @param obj is the main structure for bpf objects
  59. */
  60. static void ebpf_cachestat_disable_probe(struct cachestat_bpf *obj)
  61. {
  62. bpf_program__set_autoload(obj->progs.netdata_add_to_page_cache_lru_kprobe, false);
  63. bpf_program__set_autoload(obj->progs.netdata_mark_page_accessed_kprobe, false);
  64. bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_kprobe, false);
  65. bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_kprobe, false);
  66. bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_kprobe, false);
  67. bpf_program__set_autoload(obj->progs.netdata_mark_buffer_dirty_kprobe, false);
  68. bpf_program__set_autoload(obj->progs.netdata_release_task_kprobe, false);
  69. }
  70. /*
  71. * Disable specific probe
  72. *
  73. * Disable probes according the kernel version
  74. *
  75. * @param obj is the main structure for bpf objects
  76. */
  77. static void ebpf_cachestat_disable_specific_probe(struct cachestat_bpf *obj)
  78. {
  79. if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  80. account_page[NETDATA_CACHESTAT_FOLIO_DIRTY])) {
  81. bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_kprobe, false);
  82. bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_kprobe, false);
  83. } else if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  84. account_page[NETDATA_CACHESTAT_SET_PAGE_DIRTY])) {
  85. bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_kprobe, false);
  86. bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_kprobe, false);
  87. } else {
  88. bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_kprobe, false);
  89. bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_kprobe, false);
  90. }
  91. }
  92. /*
  93. * Disable trampoline
  94. *
  95. * Disable all trampoline to use exclusively another method.
  96. *
  97. * @param obj is the main structure for bpf objects.
  98. */
  99. static void ebpf_cachestat_disable_trampoline(struct cachestat_bpf *obj)
  100. {
  101. bpf_program__set_autoload(obj->progs.netdata_add_to_page_cache_lru_fentry, false);
  102. bpf_program__set_autoload(obj->progs.netdata_mark_page_accessed_fentry, false);
  103. bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_fentry, false);
  104. bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_fentry, false);
  105. bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_fentry, false);
  106. bpf_program__set_autoload(obj->progs.netdata_mark_buffer_dirty_fentry, false);
  107. bpf_program__set_autoload(obj->progs.netdata_release_task_fentry, false);
  108. }
  109. /*
  110. * Disable specific trampoline
  111. *
  112. * Disable trampoline according to kernel version.
  113. *
  114. * @param obj is the main structure for bpf objects.
  115. */
  116. static void ebpf_cachestat_disable_specific_trampoline(struct cachestat_bpf *obj)
  117. {
  118. if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  119. account_page[NETDATA_CACHESTAT_FOLIO_DIRTY])) {
  120. bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_fentry, false);
  121. bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_fentry, false);
  122. } else if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  123. account_page[NETDATA_CACHESTAT_SET_PAGE_DIRTY])) {
  124. bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_fentry, false);
  125. bpf_program__set_autoload(obj->progs.netdata_account_page_dirtied_fentry, false);
  126. } else {
  127. bpf_program__set_autoload(obj->progs.netdata_folio_mark_dirty_fentry, false);
  128. bpf_program__set_autoload(obj->progs.netdata_set_page_dirty_fentry, false);
  129. }
  130. }
  131. /**
  132. * Set trampoline target
  133. *
  134. * Set the targets we will monitor.
  135. *
  136. * @param obj is the main structure for bpf objects.
  137. */
  138. static inline void netdata_set_trampoline_target(struct cachestat_bpf *obj)
  139. {
  140. bpf_program__set_attach_target(obj->progs.netdata_add_to_page_cache_lru_fentry, 0,
  141. cachestat_targets[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].name);
  142. bpf_program__set_attach_target(obj->progs.netdata_mark_page_accessed_fentry, 0,
  143. cachestat_targets[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED].name);
  144. if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  145. account_page[NETDATA_CACHESTAT_FOLIO_DIRTY])) {
  146. bpf_program__set_attach_target(obj->progs.netdata_folio_mark_dirty_fentry, 0,
  147. cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
  148. } else if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  149. account_page[NETDATA_CACHESTAT_SET_PAGE_DIRTY])) {
  150. bpf_program__set_attach_target(obj->progs.netdata_set_page_dirty_fentry, 0,
  151. cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
  152. } else {
  153. bpf_program__set_attach_target(obj->progs.netdata_account_page_dirtied_fentry, 0,
  154. cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
  155. }
  156. bpf_program__set_attach_target(obj->progs.netdata_mark_buffer_dirty_fentry, 0,
  157. cachestat_targets[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY].name);
  158. bpf_program__set_attach_target(obj->progs.netdata_release_task_fentry, 0,
  159. EBPF_COMMON_FNCT_CLEAN_UP);
  160. }
  161. /**
  162. * Mount Attach Probe
  163. *
  164. * Attach probes to target
  165. *
  166. * @param obj is the main structure for bpf objects.
  167. *
  168. * @return It returns 0 on success and -1 otherwise.
  169. */
  170. static int ebpf_cachestat_attach_probe(struct cachestat_bpf *obj)
  171. {
  172. obj->links.netdata_add_to_page_cache_lru_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_add_to_page_cache_lru_kprobe,
  173. false,
  174. cachestat_targets[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].name);
  175. int ret = libbpf_get_error(obj->links.netdata_add_to_page_cache_lru_kprobe);
  176. if (ret)
  177. return -1;
  178. obj->links.netdata_mark_page_accessed_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_mark_page_accessed_kprobe,
  179. false,
  180. cachestat_targets[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED].name);
  181. ret = libbpf_get_error(obj->links.netdata_mark_page_accessed_kprobe);
  182. if (ret)
  183. return -1;
  184. if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  185. account_page[NETDATA_CACHESTAT_FOLIO_DIRTY])) {
  186. obj->links.netdata_folio_mark_dirty_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_folio_mark_dirty_kprobe,
  187. false,
  188. cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
  189. ret = libbpf_get_error(obj->links.netdata_folio_mark_dirty_kprobe);
  190. } else if (!strcmp(cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name,
  191. account_page[NETDATA_CACHESTAT_SET_PAGE_DIRTY])) {
  192. obj->links.netdata_set_page_dirty_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_set_page_dirty_kprobe,
  193. false,
  194. cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
  195. ret = libbpf_get_error(obj->links.netdata_set_page_dirty_kprobe);
  196. } else {
  197. obj->links.netdata_account_page_dirtied_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_account_page_dirtied_kprobe,
  198. false,
  199. cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name);
  200. ret = libbpf_get_error(obj->links.netdata_account_page_dirtied_kprobe);
  201. }
  202. if (ret)
  203. return -1;
  204. obj->links.netdata_mark_buffer_dirty_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_mark_buffer_dirty_kprobe,
  205. false,
  206. cachestat_targets[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY].name);
  207. ret = libbpf_get_error(obj->links.netdata_mark_buffer_dirty_kprobe);
  208. if (ret)
  209. return -1;
  210. obj->links.netdata_release_task_kprobe = bpf_program__attach_kprobe(obj->progs.netdata_release_task_kprobe,
  211. false,
  212. EBPF_COMMON_FNCT_CLEAN_UP);
  213. ret = libbpf_get_error(obj->links.netdata_release_task_kprobe);
  214. if (ret)
  215. return -1;
  216. return 0;
  217. }
  218. /**
  219. * Adjust Map Size
  220. *
  221. * Resize maps according input from users.
  222. *
  223. * @param obj is the main structure for bpf objects.
  224. * @param em structure with configuration
  225. */
  226. static void ebpf_cachestat_adjust_map(struct cachestat_bpf *obj, ebpf_module_t *em)
  227. {
  228. ebpf_update_map_size(obj->maps.cstat_pid, &cachestat_maps[NETDATA_CACHESTAT_PID_STATS],
  229. em, bpf_map__name(obj->maps.cstat_pid));
  230. ebpf_update_map_type(obj->maps.cstat_global, &cachestat_maps[NETDATA_CACHESTAT_GLOBAL_STATS]);
  231. ebpf_update_map_type(obj->maps.cstat_pid, &cachestat_maps[NETDATA_CACHESTAT_PID_STATS]);
  232. ebpf_update_map_type(obj->maps.cstat_ctrl, &cachestat_maps[NETDATA_CACHESTAT_CTRL]);
  233. }
  234. /**
  235. * Set hash tables
  236. *
  237. * Set the values for maps according the value given by kernel.
  238. *
  239. * @param obj is the main structure for bpf objects.
  240. */
  241. static void ebpf_cachestat_set_hash_tables(struct cachestat_bpf *obj)
  242. {
  243. cachestat_maps[NETDATA_CACHESTAT_GLOBAL_STATS].map_fd = bpf_map__fd(obj->maps.cstat_global);
  244. cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd = bpf_map__fd(obj->maps.cstat_pid);
  245. cachestat_maps[NETDATA_CACHESTAT_CTRL].map_fd = bpf_map__fd(obj->maps.cstat_ctrl);
  246. }
  247. /**
  248. * Disable Release Task
  249. *
  250. * Disable release task when apps is not enabled.
  251. *
  252. * @param obj is the main structure for bpf objects.
  253. */
  254. static void ebpf_cachestat_disable_release_task(struct cachestat_bpf *obj)
  255. {
  256. bpf_program__set_autoload(obj->progs.netdata_release_task_kprobe, false);
  257. bpf_program__set_autoload(obj->progs.netdata_release_task_fentry, false);
  258. }
  259. /**
  260. * Load and attach
  261. *
  262. * Load and attach the eBPF code in kernel.
  263. *
  264. * @param obj is the main structure for bpf objects.
  265. * @param em structure with configuration
  266. *
  267. * @return it returns 0 on success and -1 otherwise
  268. */
  269. static inline int ebpf_cachestat_load_and_attach(struct cachestat_bpf *obj, ebpf_module_t *em)
  270. {
  271. netdata_ebpf_targets_t *mt = em->targets;
  272. netdata_ebpf_program_loaded_t test = mt[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].mode;
  273. if (test == EBPF_LOAD_TRAMPOLINE) {
  274. ebpf_cachestat_disable_probe(obj);
  275. ebpf_cachestat_disable_specific_trampoline(obj);
  276. netdata_set_trampoline_target(obj);
  277. } else {
  278. ebpf_cachestat_disable_trampoline(obj);
  279. ebpf_cachestat_disable_specific_probe(obj);
  280. }
  281. ebpf_cachestat_adjust_map(obj, em);
  282. if (!em->apps_charts && !em->cgroup_charts)
  283. ebpf_cachestat_disable_release_task(obj);
  284. int ret = cachestat_bpf__load(obj);
  285. if (ret) {
  286. return ret;
  287. }
  288. ret = (test == EBPF_LOAD_TRAMPOLINE) ? cachestat_bpf__attach(obj) : ebpf_cachestat_attach_probe(obj);
  289. if (!ret) {
  290. ebpf_cachestat_set_hash_tables(obj);
  291. ebpf_update_controller(cachestat_maps[NETDATA_CACHESTAT_CTRL].map_fd, em);
  292. }
  293. return ret;
  294. }
  295. #endif
  296. /*****************************************************************
  297. *
  298. * FUNCTIONS TO CLOSE THE THREAD
  299. *
  300. *****************************************************************/
  301. /**
  302. * Cachestat Free
  303. *
  304. * Cleanup variables after child threads to stop
  305. *
  306. * @param ptr thread data.
  307. */
  308. static void ebpf_cachestat_free(ebpf_module_t *em)
  309. {
  310. pthread_mutex_lock(&ebpf_exit_cleanup);
  311. em->enabled = NETDATA_THREAD_EBPF_STOPPING;
  312. pthread_mutex_unlock(&ebpf_exit_cleanup);
  313. freez(cachestat_vector);
  314. freez(cachestat_values);
  315. pthread_mutex_lock(&ebpf_exit_cleanup);
  316. em->enabled = NETDATA_THREAD_EBPF_STOPPED;
  317. pthread_mutex_unlock(&ebpf_exit_cleanup);
  318. }
  319. /**
  320. * Cachestat exit.
  321. *
  322. * Cancel child and exit.
  323. *
  324. * @param ptr thread data.
  325. */
  326. static void ebpf_cachestat_exit(void *ptr)
  327. {
  328. ebpf_module_t *em = (ebpf_module_t *)ptr;
  329. ebpf_cachestat_free(em);
  330. }
  331. /*****************************************************************
  332. *
  333. * COMMON FUNCTIONS
  334. *
  335. *****************************************************************/
  336. /**
  337. * Update publish
  338. *
  339. * Update publish values before to write dimension.
  340. *
  341. * @param out structure that will receive data.
  342. * @param mpa calls for mark_page_accessed during the last second.
  343. * @param mbd calls for mark_buffer_dirty during the last second.
  344. * @param apcl calls for add_to_page_cache_lru during the last second.
  345. * @param apd calls for account_page_dirtied during the last second.
  346. */
  347. void cachestat_update_publish(netdata_publish_cachestat_t *out, uint64_t mpa, uint64_t mbd,
  348. uint64_t apcl, uint64_t apd)
  349. {
  350. // Adapted algorithm from https://github.com/iovisor/bcc/blob/master/tools/cachestat.py#L126-L138
  351. NETDATA_DOUBLE total = (NETDATA_DOUBLE) (((long long)mpa) - ((long long)mbd));
  352. if (total < 0)
  353. total = 0;
  354. NETDATA_DOUBLE misses = (NETDATA_DOUBLE) ( ((long long) apcl) - ((long long) apd) );
  355. if (misses < 0)
  356. misses = 0;
  357. // If hits are < 0, then its possible misses are overestimate due to possibly page cache read ahead adding
  358. // more pages than needed. In this case just assume misses as total and reset hits.
  359. NETDATA_DOUBLE hits = total - misses;
  360. if (hits < 0 ) {
  361. misses = total;
  362. hits = 0;
  363. }
  364. NETDATA_DOUBLE ratio = (total > 0) ? hits/total : 1;
  365. out->ratio = (long long )(ratio*100);
  366. out->hit = (long long)hits;
  367. out->miss = (long long)misses;
  368. }
  369. /**
  370. * Save previous values
  371. *
  372. * Save values used this time.
  373. *
  374. * @param publish
  375. */
  376. static void save_previous_values(netdata_publish_cachestat_t *publish) {
  377. publish->prev.mark_page_accessed = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED];
  378. publish->prev.account_page_dirtied = cachestat_hash_values[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED];
  379. publish->prev.add_to_page_cache_lru = cachestat_hash_values[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU];
  380. publish->prev.mark_buffer_dirty = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY];
  381. }
  382. /**
  383. * Calculate statistics
  384. *
  385. * @param publish the structure where we will store the data.
  386. */
  387. static void calculate_stats(netdata_publish_cachestat_t *publish) {
  388. if (!publish->prev.mark_page_accessed) {
  389. save_previous_values(publish);
  390. return;
  391. }
  392. uint64_t mpa = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_PAGE_ACCESSED] - publish->prev.mark_page_accessed;
  393. uint64_t mbd = cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY] - publish->prev.mark_buffer_dirty;
  394. uint64_t apcl = cachestat_hash_values[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU] - publish->prev.add_to_page_cache_lru;
  395. uint64_t apd = cachestat_hash_values[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED] - publish->prev.account_page_dirtied;
  396. save_previous_values(publish);
  397. // We are changing the original algorithm to have a smooth ratio.
  398. cachestat_update_publish(publish, mpa, mbd, apcl, apd);
  399. }
  400. /*****************************************************************
  401. *
  402. * APPS
  403. *
  404. *****************************************************************/
  405. /**
  406. * Apps Accumulator
  407. *
  408. * Sum all values read from kernel and store in the first address.
  409. *
  410. * @param out the vector with read values.
  411. * @param maps_per_core do I need to read all cores?
  412. */
  413. static void cachestat_apps_accumulator(netdata_cachestat_pid_t *out, int maps_per_core)
  414. {
  415. int i, end = (maps_per_core) ? ebpf_nprocs : 1;
  416. netdata_cachestat_pid_t *total = &out[0];
  417. for (i = 1; i < end; i++) {
  418. netdata_cachestat_pid_t *w = &out[i];
  419. total->account_page_dirtied += w->account_page_dirtied;
  420. total->add_to_page_cache_lru += w->add_to_page_cache_lru;
  421. total->mark_buffer_dirty += w->mark_buffer_dirty;
  422. total->mark_page_accessed += w->mark_page_accessed;
  423. }
  424. }
  425. /**
  426. * Save Pid values
  427. *
  428. * Save the current values inside the structure
  429. *
  430. * @param out vector used to plot charts
  431. * @param publish vector with values read from hash tables.
  432. */
  433. static inline void cachestat_save_pid_values(netdata_publish_cachestat_t *out, netdata_cachestat_pid_t *publish)
  434. {
  435. if (!out->current.mark_page_accessed) {
  436. memcpy(&out->current, &publish[0], sizeof(netdata_cachestat_pid_t));
  437. return;
  438. }
  439. memcpy(&out->prev, &out->current, sizeof(netdata_cachestat_pid_t));
  440. memcpy(&out->current, &publish[0], sizeof(netdata_cachestat_pid_t));
  441. }
  442. /**
  443. * Fill PID
  444. *
  445. * Fill PID structures
  446. *
  447. * @param current_pid pid that we are collecting data
  448. * @param out values read from hash tables;
  449. */
  450. static void cachestat_fill_pid(uint32_t current_pid, netdata_cachestat_pid_t *publish)
  451. {
  452. netdata_publish_cachestat_t *curr = cachestat_pid[current_pid];
  453. if (!curr) {
  454. curr = ebpf_publish_cachestat_get();
  455. cachestat_pid[current_pid] = curr;
  456. cachestat_save_pid_values(curr, publish);
  457. return;
  458. }
  459. cachestat_save_pid_values(curr, publish);
  460. }
  461. /**
  462. * Read APPS table
  463. *
  464. * Read the apps table and store data inside the structure.
  465. *
  466. * @param maps_per_core do I need to read all cores?
  467. */
  468. static void ebpf_read_cachestat_apps_table(int maps_per_core)
  469. {
  470. netdata_cachestat_pid_t *cv = cachestat_vector;
  471. uint32_t key;
  472. struct ebpf_pid_stat *pids = ebpf_root_of_pids;
  473. int fd = cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd;
  474. size_t length = sizeof(netdata_cachestat_pid_t);
  475. if (maps_per_core)
  476. length *= ebpf_nprocs;
  477. while (pids) {
  478. key = pids->pid;
  479. if (bpf_map_lookup_elem(fd, &key, cv)) {
  480. pids = pids->next;
  481. continue;
  482. }
  483. cachestat_apps_accumulator(cv, maps_per_core);
  484. cachestat_fill_pid(key, cv);
  485. // We are cleaning to avoid passing data read from one process to other.
  486. memset(cv, 0, length);
  487. pids = pids->next;
  488. }
  489. }
  490. /**
  491. * Update cgroup
  492. *
  493. * Update cgroup data based in
  494. *
  495. * @param maps_per_core do I need to read all cores?
  496. */
  497. static void ebpf_update_cachestat_cgroup(int maps_per_core)
  498. {
  499. netdata_cachestat_pid_t *cv = cachestat_vector;
  500. int fd = cachestat_maps[NETDATA_CACHESTAT_PID_STATS].map_fd;
  501. size_t length = sizeof(netdata_cachestat_pid_t);
  502. if (maps_per_core)
  503. length *= ebpf_nprocs;
  504. ebpf_cgroup_target_t *ect;
  505. pthread_mutex_lock(&mutex_cgroup_shm);
  506. for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
  507. struct pid_on_target2 *pids;
  508. for (pids = ect->pids; pids; pids = pids->next) {
  509. int pid = pids->pid;
  510. netdata_cachestat_pid_t *out = &pids->cachestat;
  511. if (likely(cachestat_pid) && cachestat_pid[pid]) {
  512. netdata_publish_cachestat_t *in = cachestat_pid[pid];
  513. memcpy(out, &in->current, sizeof(netdata_cachestat_pid_t));
  514. } else {
  515. memset(cv, 0, length);
  516. if (bpf_map_lookup_elem(fd, &pid, cv)) {
  517. continue;
  518. }
  519. cachestat_apps_accumulator(cv, maps_per_core);
  520. memcpy(out, cv, sizeof(netdata_cachestat_pid_t));
  521. }
  522. }
  523. }
  524. pthread_mutex_unlock(&mutex_cgroup_shm);
  525. }
  526. /**
  527. * Create apps charts
  528. *
  529. * Call ebpf_create_chart to create the charts on apps submenu.
  530. *
  531. * @param em a pointer to the structure with the default values.
  532. */
  533. void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *ptr)
  534. {
  535. struct ebpf_target *root = ptr;
  536. ebpf_create_charts_on_apps(NETDATA_CACHESTAT_HIT_RATIO_CHART,
  537. "Hit ratio",
  538. EBPF_COMMON_DIMENSION_PERCENTAGE,
  539. NETDATA_CACHESTAT_SUBMENU,
  540. NETDATA_EBPF_CHART_TYPE_LINE,
  541. 20090,
  542. ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
  543. root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  544. ebpf_create_charts_on_apps(NETDATA_CACHESTAT_DIRTY_CHART,
  545. "Number of dirty pages",
  546. EBPF_CACHESTAT_DIMENSION_PAGE,
  547. NETDATA_CACHESTAT_SUBMENU,
  548. NETDATA_EBPF_CHART_TYPE_STACKED,
  549. 20091,
  550. ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX],
  551. root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  552. ebpf_create_charts_on_apps(NETDATA_CACHESTAT_HIT_CHART,
  553. "Number of accessed files",
  554. EBPF_CACHESTAT_DIMENSION_HITS,
  555. NETDATA_CACHESTAT_SUBMENU,
  556. NETDATA_EBPF_CHART_TYPE_STACKED,
  557. 20092,
  558. ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
  559. root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  560. ebpf_create_charts_on_apps(NETDATA_CACHESTAT_MISSES_CHART,
  561. "Files out of page cache",
  562. EBPF_CACHESTAT_DIMENSION_MISSES,
  563. NETDATA_CACHESTAT_SUBMENU,
  564. NETDATA_EBPF_CHART_TYPE_STACKED,
  565. 20093,
  566. ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
  567. root, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  568. em->apps_charts |= NETDATA_EBPF_APPS_FLAG_CHART_CREATED;
  569. }
  570. /*****************************************************************
  571. *
  572. * MAIN LOOP
  573. *
  574. *****************************************************************/
  575. /**
  576. * Read global counter
  577. *
  578. * Read the table with number of calls for all functions
  579. *
  580. * @param maps_per_core do I need to read all cores?
  581. */
  582. static void ebpf_cachestat_read_global_table(int maps_per_core)
  583. {
  584. uint32_t idx;
  585. netdata_idx_t *val = cachestat_hash_values;
  586. netdata_idx_t *stored = cachestat_values;
  587. int fd = cachestat_maps[NETDATA_CACHESTAT_GLOBAL_STATS].map_fd;
  588. for (idx = NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU; idx < NETDATA_CACHESTAT_END; idx++) {
  589. if (!bpf_map_lookup_elem(fd, &idx, stored)) {
  590. int i;
  591. int end = (maps_per_core) ? ebpf_nprocs: 1;
  592. netdata_idx_t total = 0;
  593. for (i = 0; i < end; i++)
  594. total += stored[i];
  595. val[idx] = total;
  596. }
  597. }
  598. }
  599. /**
  600. * Send global
  601. *
  602. * Send global charts to Netdata
  603. */
  604. static void cachestat_send_global(netdata_publish_cachestat_t *publish)
  605. {
  606. calculate_stats(publish);
  607. netdata_publish_syscall_t *ptr = cachestat_counter_publish_aggregated;
  608. ebpf_one_dimension_write_charts(
  609. NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART, ptr[NETDATA_CACHESTAT_IDX_RATIO].dimension,
  610. publish->ratio);
  611. ebpf_one_dimension_write_charts(
  612. NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART, ptr[NETDATA_CACHESTAT_IDX_DIRTY].dimension,
  613. cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY]);
  614. ebpf_one_dimension_write_charts(
  615. NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART, ptr[NETDATA_CACHESTAT_IDX_HIT].dimension, publish->hit);
  616. ebpf_one_dimension_write_charts(
  617. NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART, ptr[NETDATA_CACHESTAT_IDX_MISS].dimension,
  618. publish->miss);
  619. }
  620. /**
  621. * Cachestat sum PIDs
  622. *
  623. * Sum values for all PIDs associated to a group
  624. *
  625. * @param publish output structure.
  626. * @param root structure with listed IPs
  627. */
  628. void ebpf_cachestat_sum_pids(netdata_publish_cachestat_t *publish, struct ebpf_pid_on_target *root)
  629. {
  630. memcpy(&publish->prev, &publish->current,sizeof(publish->current));
  631. memset(&publish->current, 0, sizeof(publish->current));
  632. netdata_cachestat_pid_t *dst = &publish->current;
  633. while (root) {
  634. int32_t pid = root->pid;
  635. netdata_publish_cachestat_t *w = cachestat_pid[pid];
  636. if (w) {
  637. netdata_cachestat_pid_t *src = &w->current;
  638. dst->account_page_dirtied += src->account_page_dirtied;
  639. dst->add_to_page_cache_lru += src->add_to_page_cache_lru;
  640. dst->mark_buffer_dirty += src->mark_buffer_dirty;
  641. dst->mark_page_accessed += src->mark_page_accessed;
  642. }
  643. root = root->next;
  644. }
  645. }
  646. /**
  647. * Send data to Netdata calling auxiliary functions.
  648. *
  649. * @param root the target list.
  650. */
  651. void ebpf_cache_send_apps_data(struct ebpf_target *root)
  652. {
  653. struct ebpf_target *w;
  654. collected_number value;
  655. write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_HIT_RATIO_CHART);
  656. for (w = root; w; w = w->next) {
  657. if (unlikely(w->exposed && w->processes)) {
  658. ebpf_cachestat_sum_pids(&w->cachestat, w->root_pid);
  659. netdata_cachestat_pid_t *current = &w->cachestat.current;
  660. netdata_cachestat_pid_t *prev = &w->cachestat.prev;
  661. uint64_t mpa = current->mark_page_accessed - prev->mark_page_accessed;
  662. uint64_t mbd = current->mark_buffer_dirty - prev->mark_buffer_dirty;
  663. w->cachestat.dirty = mbd;
  664. uint64_t apcl = current->add_to_page_cache_lru - prev->add_to_page_cache_lru;
  665. uint64_t apd = current->account_page_dirtied - prev->account_page_dirtied;
  666. cachestat_update_publish(&w->cachestat, mpa, mbd, apcl, apd);
  667. value = (collected_number) w->cachestat.ratio;
  668. // Here we are using different approach to have a chart more smooth
  669. write_chart_dimension(w->name, value);
  670. }
  671. }
  672. write_end_chart();
  673. write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_DIRTY_CHART);
  674. for (w = root; w; w = w->next) {
  675. if (unlikely(w->exposed && w->processes)) {
  676. value = (collected_number) w->cachestat.dirty;
  677. write_chart_dimension(w->name, value);
  678. }
  679. }
  680. write_end_chart();
  681. write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_HIT_CHART);
  682. for (w = root; w; w = w->next) {
  683. if (unlikely(w->exposed && w->processes)) {
  684. value = (collected_number) w->cachestat.hit;
  685. write_chart_dimension(w->name, value);
  686. }
  687. }
  688. write_end_chart();
  689. write_begin_chart(NETDATA_APPS_FAMILY, NETDATA_CACHESTAT_MISSES_CHART);
  690. for (w = root; w; w = w->next) {
  691. if (unlikely(w->exposed && w->processes)) {
  692. value = (collected_number) w->cachestat.miss;
  693. write_chart_dimension(w->name, value);
  694. }
  695. }
  696. write_end_chart();
  697. }
  698. /**
  699. * Cachestat sum PIDs
  700. *
  701. * Sum values for all PIDs associated to a group
  702. *
  703. * @param publish output structure.
  704. * @param root structure with listed IPs
  705. */
  706. void ebpf_cachestat_sum_cgroup_pids(netdata_publish_cachestat_t *publish, struct pid_on_target2 *root)
  707. {
  708. memcpy(&publish->prev, &publish->current,sizeof(publish->current));
  709. memset(&publish->current, 0, sizeof(publish->current));
  710. netdata_cachestat_pid_t *dst = &publish->current;
  711. while (root) {
  712. netdata_cachestat_pid_t *src = &root->cachestat;
  713. dst->account_page_dirtied += src->account_page_dirtied;
  714. dst->add_to_page_cache_lru += src->add_to_page_cache_lru;
  715. dst->mark_buffer_dirty += src->mark_buffer_dirty;
  716. dst->mark_page_accessed += src->mark_page_accessed;
  717. root = root->next;
  718. }
  719. }
  720. /**
  721. * Calc chart values
  722. *
  723. * Do necessary math to plot charts.
  724. */
  725. void ebpf_cachestat_calc_chart_values()
  726. {
  727. ebpf_cgroup_target_t *ect;
  728. for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
  729. ebpf_cachestat_sum_cgroup_pids(&ect->publish_cachestat, ect->pids);
  730. netdata_cachestat_pid_t *current = &ect->publish_cachestat.current;
  731. netdata_cachestat_pid_t *prev = &ect->publish_cachestat.prev;
  732. uint64_t mpa = current->mark_page_accessed - prev->mark_page_accessed;
  733. uint64_t mbd = current->mark_buffer_dirty - prev->mark_buffer_dirty;
  734. ect->publish_cachestat.dirty = mbd;
  735. uint64_t apcl = current->add_to_page_cache_lru - prev->add_to_page_cache_lru;
  736. uint64_t apd = current->account_page_dirtied - prev->account_page_dirtied;
  737. cachestat_update_publish(&ect->publish_cachestat, mpa, mbd, apcl, apd);
  738. }
  739. }
  740. /**
  741. * Create Systemd cachestat Charts
  742. *
  743. * Create charts when systemd is enabled
  744. *
  745. * @param update_every value to overwrite the update frequency set by the server.
  746. **/
  747. static void ebpf_create_systemd_cachestat_charts(int update_every)
  748. {
  749. ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_HIT_RATIO_CHART,
  750. "Hit ratio",
  751. EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_SUBMENU,
  752. NETDATA_EBPF_CHART_TYPE_LINE, 21100,
  753. ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
  754. NETDATA_SYSTEMD_CACHESTAT_HIT_RATIO_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
  755. update_every);
  756. ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_DIRTY_CHART,
  757. "Number of dirty pages",
  758. EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_SUBMENU,
  759. NETDATA_EBPF_CHART_TYPE_LINE, 21101,
  760. ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
  761. NETDATA_SYSTEMD_CACHESTAT_MODIFIED_CACHE_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
  762. update_every);
  763. ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_HIT_CHART, "Number of accessed files",
  764. EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU,
  765. NETDATA_EBPF_CHART_TYPE_LINE, 21102,
  766. ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
  767. NETDATA_SYSTEMD_CACHESTAT_HIT_FILE_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
  768. update_every);
  769. ebpf_create_charts_on_systemd(NETDATA_CACHESTAT_MISSES_CHART, "Files out of page cache",
  770. EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_SUBMENU,
  771. NETDATA_EBPF_CHART_TYPE_LINE, 21103,
  772. ebpf_algorithms[NETDATA_EBPF_ABSOLUTE_IDX],
  773. NETDATA_SYSTEMD_CACHESTAT_MISS_FILES_CONTEXT, NETDATA_EBPF_MODULE_NAME_CACHESTAT,
  774. update_every);
  775. }
  776. /**
  777. * Send Cache Stat charts
  778. *
  779. * Send collected data to Netdata.
  780. */
  781. static void ebpf_send_systemd_cachestat_charts()
  782. {
  783. ebpf_cgroup_target_t *ect;
  784. write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_HIT_RATIO_CHART);
  785. for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
  786. if (unlikely(ect->systemd) && unlikely(ect->updated)) {
  787. write_chart_dimension(ect->name, (long long)ect->publish_cachestat.ratio);
  788. }
  789. }
  790. write_end_chart();
  791. write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_DIRTY_CHART);
  792. for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
  793. if (unlikely(ect->systemd) && unlikely(ect->updated)) {
  794. write_chart_dimension(ect->name, (long long)ect->publish_cachestat.dirty);
  795. }
  796. }
  797. write_end_chart();
  798. write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_HIT_CHART);
  799. for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
  800. if (unlikely(ect->systemd) && unlikely(ect->updated)) {
  801. write_chart_dimension(ect->name, (long long)ect->publish_cachestat.hit);
  802. }
  803. }
  804. write_end_chart();
  805. write_begin_chart(NETDATA_SERVICE_FAMILY, NETDATA_CACHESTAT_MISSES_CHART);
  806. for (ect = ebpf_cgroup_pids; ect; ect = ect->next) {
  807. if (unlikely(ect->systemd) && unlikely(ect->updated)) {
  808. write_chart_dimension(ect->name, (long long)ect->publish_cachestat.miss);
  809. }
  810. }
  811. write_end_chart();
  812. }
  813. /**
  814. * Send Directory Cache charts
  815. *
  816. * Send collected data to Netdata.
  817. */
  818. static void ebpf_send_specific_cachestat_data(char *type, netdata_publish_cachestat_t *npc)
  819. {
  820. write_begin_chart(type, NETDATA_CACHESTAT_HIT_RATIO_CHART);
  821. write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_RATIO].name, (long long)npc->ratio);
  822. write_end_chart();
  823. write_begin_chart(type, NETDATA_CACHESTAT_DIRTY_CHART);
  824. write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_DIRTY].name, (long long)npc->dirty);
  825. write_end_chart();
  826. write_begin_chart(type, NETDATA_CACHESTAT_HIT_CHART);
  827. write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_HIT].name, (long long)npc->hit);
  828. write_end_chart();
  829. write_begin_chart(type, NETDATA_CACHESTAT_MISSES_CHART);
  830. write_chart_dimension(cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_MISS].name, (long long)npc->miss);
  831. write_end_chart();
  832. }
  833. /**
  834. * Create specific cache Stat charts
  835. *
  836. * Create charts for cgroup/application.
  837. *
  838. * @param type the chart type.
  839. * @param update_every value to overwrite the update frequency set by the server.
  840. */
  841. static void ebpf_create_specific_cachestat_charts(char *type, int update_every)
  842. {
  843. ebpf_create_chart(type, NETDATA_CACHESTAT_HIT_RATIO_CHART,
  844. "Hit ratio",
  845. EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_CGROUP_SUBMENU,
  846. NETDATA_CGROUP_CACHESTAT_HIT_RATIO_CONTEXT,
  847. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5200,
  848. ebpf_create_global_dimension,
  849. cachestat_counter_publish_aggregated, 1, update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  850. ebpf_create_chart(type, NETDATA_CACHESTAT_DIRTY_CHART,
  851. "Number of dirty pages",
  852. EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_CGROUP_SUBMENU,
  853. NETDATA_CGROUP_CACHESTAT_MODIFIED_CACHE_CONTEXT,
  854. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5201,
  855. ebpf_create_global_dimension,
  856. &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_DIRTY], 1,
  857. update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  858. ebpf_create_chart(type, NETDATA_CACHESTAT_HIT_CHART,
  859. "Number of accessed files",
  860. EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_CGROUP_SUBMENU,
  861. NETDATA_CGROUP_CACHESTAT_HIT_FILES_CONTEXT,
  862. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5202,
  863. ebpf_create_global_dimension,
  864. &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_HIT], 1,
  865. update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  866. ebpf_create_chart(type, NETDATA_CACHESTAT_MISSES_CHART,
  867. "Files out of page cache",
  868. EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_CGROUP_SUBMENU,
  869. NETDATA_CGROUP_CACHESTAT_MISS_FILES_CONTEXT,
  870. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5203,
  871. ebpf_create_global_dimension,
  872. &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_MISS], 1,
  873. update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  874. }
  875. /**
  876. * Obsolete specific cache stat charts
  877. *
  878. * Obsolete charts for cgroup/application.
  879. *
  880. * @param type the chart type.
  881. * @param update_every value to overwrite the update frequency set by the server.
  882. */
  883. static void ebpf_obsolete_specific_cachestat_charts(char *type, int update_every)
  884. {
  885. ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_HIT_RATIO_CHART,
  886. "Hit ratio",
  887. EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_SUBMENU,
  888. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_HIT_RATIO_CONTEXT,
  889. NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5200, update_every);
  890. ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_DIRTY_CHART,
  891. "Number of dirty pages",
  892. EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_SUBMENU,
  893. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_MODIFIED_CACHE_CONTEXT,
  894. NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5201, update_every);
  895. ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_HIT_CHART,
  896. "Number of accessed files",
  897. EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU,
  898. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_HIT_FILES_CONTEXT,
  899. NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5202, update_every);
  900. ebpf_write_chart_obsolete(type, NETDATA_CACHESTAT_MISSES_CHART,
  901. "Files out of page cache",
  902. EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_SUBMENU,
  903. NETDATA_EBPF_CHART_TYPE_LINE, NETDATA_CGROUP_CACHESTAT_MISS_FILES_CONTEXT,
  904. NETDATA_CHART_PRIO_CGROUPS_CONTAINERS + 5203, update_every);
  905. }
  906. /**
  907. * Send data to Netdata calling auxiliary functions.
  908. *
  909. * @param update_every value to overwrite the update frequency set by the server.
  910. */
  911. void ebpf_cachestat_send_cgroup_data(int update_every)
  912. {
  913. if (!ebpf_cgroup_pids)
  914. return;
  915. pthread_mutex_lock(&mutex_cgroup_shm);
  916. ebpf_cgroup_target_t *ect;
  917. ebpf_cachestat_calc_chart_values();
  918. int has_systemd = shm_ebpf_cgroup.header->systemd_enabled;
  919. if (has_systemd) {
  920. if (send_cgroup_chart) {
  921. ebpf_create_systemd_cachestat_charts(update_every);
  922. }
  923. ebpf_send_systemd_cachestat_charts();
  924. }
  925. for (ect = ebpf_cgroup_pids; ect ; ect = ect->next) {
  926. if (ect->systemd)
  927. continue;
  928. if (!(ect->flags & NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART) && ect->updated) {
  929. ebpf_create_specific_cachestat_charts(ect->name, update_every);
  930. ect->flags |= NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART;
  931. }
  932. if (ect->flags & NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART) {
  933. if (ect->updated) {
  934. ebpf_send_specific_cachestat_data(ect->name, &ect->publish_cachestat);
  935. } else {
  936. ebpf_obsolete_specific_cachestat_charts(ect->name, update_every);
  937. ect->flags &= ~NETDATA_EBPF_CGROUP_HAS_CACHESTAT_CHART;
  938. }
  939. }
  940. }
  941. pthread_mutex_unlock(&mutex_cgroup_shm);
  942. }
  943. /**
  944. * Main loop for this collector.
  945. */
  946. static void cachestat_collector(ebpf_module_t *em)
  947. {
  948. netdata_publish_cachestat_t publish;
  949. memset(&publish, 0, sizeof(publish));
  950. int cgroups = em->cgroup_charts;
  951. int update_every = em->update_every;
  952. int maps_per_core = em->maps_per_core;
  953. heartbeat_t hb;
  954. heartbeat_init(&hb);
  955. int counter = update_every - 1;
  956. //This will be cancelled by its parent
  957. while (!ebpf_exit_plugin) {
  958. (void)heartbeat_next(&hb, USEC_PER_SEC);
  959. if (ebpf_exit_plugin || ++counter != update_every)
  960. continue;
  961. counter = 0;
  962. netdata_apps_integration_flags_t apps = em->apps_charts;
  963. ebpf_cachestat_read_global_table(maps_per_core);
  964. pthread_mutex_lock(&collect_data_mutex);
  965. if (apps)
  966. ebpf_read_cachestat_apps_table(maps_per_core);
  967. if (cgroups)
  968. ebpf_update_cachestat_cgroup(maps_per_core);
  969. pthread_mutex_lock(&lock);
  970. cachestat_send_global(&publish);
  971. if (apps & NETDATA_EBPF_APPS_FLAG_CHART_CREATED)
  972. ebpf_cache_send_apps_data(apps_groups_root_target);
  973. #ifdef NETDATA_DEV_MODE
  974. if (ebpf_aral_cachestat_pid)
  975. ebpf_send_data_aral_chart(ebpf_aral_cachestat_pid, em);
  976. #endif
  977. if (cgroups)
  978. ebpf_cachestat_send_cgroup_data(update_every);
  979. pthread_mutex_unlock(&lock);
  980. pthread_mutex_unlock(&collect_data_mutex);
  981. }
  982. }
  983. /*****************************************************************
  984. *
  985. * INITIALIZE THREAD
  986. *
  987. *****************************************************************/
  988. /**
  989. * Create global charts
  990. *
  991. * Call ebpf_create_chart to create the charts for the collector.
  992. *
  993. * @param em a pointer to `struct ebpf_module`
  994. */
  995. static void ebpf_create_memory_charts(ebpf_module_t *em)
  996. {
  997. ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART,
  998. "Hit ratio",
  999. EBPF_COMMON_DIMENSION_PERCENTAGE, NETDATA_CACHESTAT_SUBMENU,
  1000. NULL,
  1001. NETDATA_EBPF_CHART_TYPE_LINE,
  1002. 21100,
  1003. ebpf_create_global_dimension,
  1004. cachestat_counter_publish_aggregated, 1, em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  1005. ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART,
  1006. "Number of dirty pages",
  1007. EBPF_CACHESTAT_DIMENSION_PAGE, NETDATA_CACHESTAT_SUBMENU,
  1008. NULL,
  1009. NETDATA_EBPF_CHART_TYPE_LINE,
  1010. 21101,
  1011. ebpf_create_global_dimension,
  1012. &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_DIRTY], 1,
  1013. em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  1014. ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART,
  1015. "Number of accessed files",
  1016. EBPF_CACHESTAT_DIMENSION_HITS, NETDATA_CACHESTAT_SUBMENU,
  1017. NULL,
  1018. NETDATA_EBPF_CHART_TYPE_LINE,
  1019. 21102,
  1020. ebpf_create_global_dimension,
  1021. &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_HIT], 1,
  1022. em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  1023. ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART,
  1024. "Files out of page cache",
  1025. EBPF_CACHESTAT_DIMENSION_MISSES, NETDATA_CACHESTAT_SUBMENU,
  1026. NULL,
  1027. NETDATA_EBPF_CHART_TYPE_LINE,
  1028. 21103,
  1029. ebpf_create_global_dimension,
  1030. &cachestat_counter_publish_aggregated[NETDATA_CACHESTAT_IDX_MISS], 1,
  1031. em->update_every, NETDATA_EBPF_MODULE_NAME_CACHESTAT);
  1032. fflush(stdout);
  1033. }
  1034. /**
  1035. * Allocate vectors used with this thread.
  1036. *
  1037. * We are not testing the return, because callocz does this and shutdown the software
  1038. * case it was not possible to allocate.
  1039. *
  1040. * @param apps is apps enabled?
  1041. */
  1042. static void ebpf_cachestat_allocate_global_vectors(int apps)
  1043. {
  1044. if (apps) {
  1045. cachestat_pid = callocz((size_t)pid_max, sizeof(netdata_publish_cachestat_t *));
  1046. ebpf_cachestat_aral_init();
  1047. cachestat_vector = callocz((size_t)ebpf_nprocs, sizeof(netdata_cachestat_pid_t));
  1048. }
  1049. cachestat_values = callocz((size_t)ebpf_nprocs, sizeof(netdata_idx_t));
  1050. memset(cachestat_hash_values, 0, NETDATA_CACHESTAT_END * sizeof(netdata_idx_t));
  1051. memset(cachestat_counter_aggregated_data, 0, NETDATA_CACHESTAT_END * sizeof(netdata_syscall_stat_t));
  1052. memset(cachestat_counter_publish_aggregated, 0, NETDATA_CACHESTAT_END * sizeof(netdata_publish_syscall_t));
  1053. }
  1054. /*****************************************************************
  1055. *
  1056. * MAIN THREAD
  1057. *
  1058. *****************************************************************/
  1059. /**
  1060. * Update Internal value
  1061. *
  1062. * Update values used during runtime.
  1063. *
  1064. * @return It returns 0 when one of the functions is present and -1 otherwise.
  1065. */
  1066. static int ebpf_cachestat_set_internal_value()
  1067. {
  1068. ebpf_addresses_t address = {.function = NULL, .hash = 0, .addr = 0};
  1069. int i;
  1070. for (i = 0; i < NETDATA_CACHESTAT_ACCOUNT_DIRTY_END ; i++) {
  1071. address.function = account_page[i];
  1072. ebpf_load_addresses(&address, -1);
  1073. if (address.addr)
  1074. break;
  1075. }
  1076. if (!address.addr) {
  1077. error("%s cachestat.", NETDATA_EBPF_DEFAULT_FNT_NOT_FOUND);
  1078. return -1;
  1079. }
  1080. cachestat_targets[NETDATA_KEY_CALLS_ACCOUNT_PAGE_DIRTIED].name = address.function;
  1081. return 0;
  1082. }
  1083. /*
  1084. * Load BPF
  1085. *
  1086. * Load BPF files.
  1087. *
  1088. * @param em the structure with configuration
  1089. */
  1090. static int ebpf_cachestat_load_bpf(ebpf_module_t *em)
  1091. {
  1092. #ifdef LIBBPF_MAJOR_VERSION
  1093. ebpf_define_map_type(cachestat_maps, em->maps_per_core, running_on_kernel);
  1094. #endif
  1095. int ret = 0;
  1096. ebpf_adjust_apps_cgroup(em, em->targets[NETDATA_KEY_CALLS_ADD_TO_PAGE_CACHE_LRU].mode);
  1097. if (em->load & EBPF_LOAD_LEGACY) {
  1098. em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
  1099. if (!em->probe_links) {
  1100. ret = -1;
  1101. }
  1102. }
  1103. #ifdef LIBBPF_MAJOR_VERSION
  1104. else {
  1105. cachestat_bpf_obj = cachestat_bpf__open();
  1106. if (!cachestat_bpf_obj)
  1107. ret = -1;
  1108. else
  1109. ret = ebpf_cachestat_load_and_attach(cachestat_bpf_obj, em);
  1110. }
  1111. #endif
  1112. if (ret)
  1113. error("%s %s", EBPF_DEFAULT_ERROR_MSG, em->thread_name);
  1114. return ret;
  1115. }
  1116. /**
  1117. * Cachestat thread
  1118. *
  1119. * Thread used to make cachestat thread
  1120. *
  1121. * @param ptr a pointer to `struct ebpf_module`
  1122. *
  1123. * @return It always return NULL
  1124. */
  1125. void *ebpf_cachestat_thread(void *ptr)
  1126. {
  1127. netdata_thread_cleanup_push(ebpf_cachestat_exit, ptr);
  1128. ebpf_module_t *em = (ebpf_module_t *)ptr;
  1129. em->maps = cachestat_maps;
  1130. ebpf_update_pid_table(&cachestat_maps[NETDATA_CACHESTAT_PID_STATS], em);
  1131. if (ebpf_cachestat_set_internal_value()) {
  1132. goto endcachestat;
  1133. }
  1134. #ifdef LIBBPF_MAJOR_VERSION
  1135. ebpf_adjust_thread_load(em, default_btf);
  1136. #endif
  1137. if (ebpf_cachestat_load_bpf(em)) {
  1138. goto endcachestat;
  1139. }
  1140. ebpf_cachestat_allocate_global_vectors(em->apps_charts);
  1141. int algorithms[NETDATA_CACHESTAT_END] = {
  1142. NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_ABSOLUTE_IDX, NETDATA_EBPF_ABSOLUTE_IDX
  1143. };
  1144. ebpf_global_labels(cachestat_counter_aggregated_data, cachestat_counter_publish_aggregated,
  1145. cachestat_counter_dimension_name, cachestat_counter_dimension_name,
  1146. algorithms, NETDATA_CACHESTAT_END);
  1147. pthread_mutex_lock(&lock);
  1148. ebpf_update_stats(&plugin_statistics, em);
  1149. ebpf_update_kernel_memory_with_vector(&plugin_statistics, em->maps);
  1150. ebpf_create_memory_charts(em);
  1151. #ifdef NETDATA_DEV_MODE
  1152. if (ebpf_aral_cachestat_pid)
  1153. ebpf_statistic_create_aral_chart(NETDATA_EBPF_CACHESTAT_ARAL_NAME, em);
  1154. #endif
  1155. pthread_mutex_unlock(&lock);
  1156. cachestat_collector(em);
  1157. endcachestat:
  1158. ebpf_update_disabled_plugin_stats(em);
  1159. netdata_thread_cleanup_pop(1);
  1160. return NULL;
  1161. }