epoll.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545
  1. /*
  2. * Copyright 2000-2007 Niels Provos <provos@citi.umich.edu>
  3. * Copyright 2007-2012 Niels Provos, Nick Mathewson
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions
  7. * are met:
  8. * 1. Redistributions of source code must retain the above copyright
  9. * notice, this list of conditions and the following disclaimer.
  10. * 2. Redistributions in binary form must reproduce the above copyright
  11. * notice, this list of conditions and the following disclaimer in the
  12. * documentation and/or other materials provided with the distribution.
  13. * 3. The name of the author may not be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  17. * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  18. * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  19. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  20. * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  21. * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  22. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  23. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  25. * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. */
  27. #include "event2/event-config.h"
  28. #include "evconfig-private.h"
  29. #ifdef EVENT__HAVE_EPOLL
  30. #include <stdint.h>
  31. #include <sys/types.h>
  32. #include <sys/resource.h>
  33. #ifdef EVENT__HAVE_SYS_TIME_H
  34. #include <sys/time.h>
  35. #endif
  36. #include <sys/queue.h>
  37. #include <sys/epoll.h>
  38. #include <signal.h>
  39. #include <limits.h>
  40. #include <stdio.h>
  41. #include <stdlib.h>
  42. #include <string.h>
  43. #include <unistd.h>
  44. #include <errno.h>
  45. #ifdef EVENT__HAVE_FCNTL_H
  46. #include <fcntl.h>
  47. #endif
  48. #ifdef EVENT__HAVE_SYS_TIMERFD_H
  49. #include <sys/timerfd.h>
  50. #endif
  51. #include "event-internal.h"
  52. #include "evsignal-internal.h"
  53. #include "event2/thread.h"
  54. #include "evthread-internal.h"
  55. #include "log-internal.h"
  56. #include "evmap-internal.h"
  57. #include "changelist-internal.h"
  58. #include "time-internal.h"
  59. /* Since Linux 2.6.17, epoll is able to report about peer half-closed connection
  60. using special EPOLLRDHUP flag on a read event.
  61. */
  62. #if !defined(EPOLLRDHUP)
  63. #define EPOLLRDHUP 0
  64. #define EARLY_CLOSE_IF_HAVE_RDHUP 0
  65. #else
  66. #define EARLY_CLOSE_IF_HAVE_RDHUP EV_FEATURE_EARLY_CLOSE
  67. #endif
  68. #include "epolltable-internal.h"
  69. #if defined(EVENT__HAVE_SYS_TIMERFD_H) && \
  70. defined(EVENT__HAVE_TIMERFD_CREATE) && \
  71. defined(HAVE_POSIX_MONOTONIC) && defined(TFD_NONBLOCK) && \
  72. defined(TFD_CLOEXEC)
  73. /* Note that we only use timerfd if TFD_NONBLOCK and TFD_CLOEXEC are available
  74. and working. This means that we can't support it on 2.6.25 (where timerfd
  75. was introduced) or 2.6.26, since 2.6.27 introduced those flags.
  76. */
  77. #define USING_TIMERFD
  78. #endif
  79. struct epollop {
  80. struct epoll_event *events;
  81. int nevents;
  82. int epfd;
  83. #ifdef USING_TIMERFD
  84. int timerfd;
  85. #endif
  86. };
  87. static void *epoll_init(struct event_base *);
  88. static int epoll_dispatch(struct event_base *, struct timeval *);
  89. static void epoll_dealloc(struct event_base *);
  90. static const struct eventop epollops_changelist = {
  91. "epoll (with changelist)",
  92. epoll_init,
  93. event_changelist_add_,
  94. event_changelist_del_,
  95. epoll_dispatch,
  96. epoll_dealloc,
  97. 1, /* need reinit */
  98. EV_FEATURE_ET|EV_FEATURE_O1| EARLY_CLOSE_IF_HAVE_RDHUP,
  99. EVENT_CHANGELIST_FDINFO_SIZE
  100. };
  101. static int epoll_nochangelist_add(struct event_base *base, evutil_socket_t fd,
  102. short old, short events, void *p);
  103. static int epoll_nochangelist_del(struct event_base *base, evutil_socket_t fd,
  104. short old, short events, void *p);
  105. const struct eventop epollops = {
  106. "epoll",
  107. epoll_init,
  108. epoll_nochangelist_add,
  109. epoll_nochangelist_del,
  110. epoll_dispatch,
  111. epoll_dealloc,
  112. 1, /* need reinit */
  113. EV_FEATURE_ET|EV_FEATURE_O1|EV_FEATURE_EARLY_CLOSE,
  114. 0
  115. };
  116. #define INITIAL_NEVENT 32
  117. #define MAX_NEVENT 4096
  118. /* On Linux kernels at least up to 2.6.24.4, epoll can't handle timeout
  119. * values bigger than (LONG_MAX - 999ULL)/HZ. HZ in the wild can be
  120. * as big as 1000, and LONG_MAX can be as small as (1<<31)-1, so the
  121. * largest number of msec we can support here is 2147482. Let's
  122. * round that down by 47 seconds.
  123. */
  124. #define MAX_EPOLL_TIMEOUT_MSEC (35*60*1000)
  125. static void *
  126. epoll_init(struct event_base *base)
  127. {
  128. int epfd = -1;
  129. struct epollop *epollop;
  130. #ifdef EVENT__HAVE_EPOLL_CREATE1
  131. /* First, try the shiny new epoll_create1 interface, if we have it. */
  132. epfd = epoll_create1(EPOLL_CLOEXEC);
  133. #endif
  134. if (epfd == -1) {
  135. /* Initialize the kernel queue using the old interface. (The
  136. size field is ignored since 2.6.8.) */
  137. if ((epfd = epoll_create(32000)) == -1) {
  138. if (errno != ENOSYS)
  139. event_warn("epoll_create");
  140. return (NULL);
  141. }
  142. evutil_make_socket_closeonexec(epfd);
  143. }
  144. if (!(epollop = mm_calloc(1, sizeof(struct epollop)))) {
  145. close(epfd);
  146. return (NULL);
  147. }
  148. epollop->epfd = epfd;
  149. /* Initialize fields */
  150. epollop->events = mm_calloc(INITIAL_NEVENT, sizeof(struct epoll_event));
  151. if (epollop->events == NULL) {
  152. mm_free(epollop);
  153. close(epfd);
  154. return (NULL);
  155. }
  156. epollop->nevents = INITIAL_NEVENT;
  157. if ((base->flags & EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST) != 0 ||
  158. ((base->flags & EVENT_BASE_FLAG_IGNORE_ENV) == 0 &&
  159. evutil_getenv_("EVENT_EPOLL_USE_CHANGELIST") != NULL)) {
  160. base->evsel = &epollops_changelist;
  161. }
  162. #ifdef USING_TIMERFD
  163. /*
  164. The epoll interface ordinarily gives us one-millisecond precision,
  165. so on Linux it makes perfect sense to use the CLOCK_MONOTONIC_COARSE
  166. timer. But when the user has set the new PRECISE_TIMER flag for an
  167. event_base, we can try to use timerfd to give them finer granularity.
  168. */
  169. if ((base->flags & EVENT_BASE_FLAG_PRECISE_TIMER) &&
  170. base->monotonic_timer.monotonic_clock == CLOCK_MONOTONIC) {
  171. int fd;
  172. fd = epollop->timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK|TFD_CLOEXEC);
  173. if (epollop->timerfd >= 0) {
  174. struct epoll_event epev;
  175. memset(&epev, 0, sizeof(epev));
  176. epev.data.fd = epollop->timerfd;
  177. epev.events = EPOLLIN;
  178. if (epoll_ctl(epollop->epfd, EPOLL_CTL_ADD, fd, &epev) < 0) {
  179. event_warn("epoll_ctl(timerfd)");
  180. close(fd);
  181. epollop->timerfd = -1;
  182. }
  183. } else {
  184. if (errno != EINVAL && errno != ENOSYS) {
  185. /* These errors probably mean that we were
  186. * compiled with timerfd/TFD_* support, but
  187. * we're running on a kernel that lacks those.
  188. */
  189. event_warn("timerfd_create");
  190. }
  191. epollop->timerfd = -1;
  192. }
  193. } else {
  194. epollop->timerfd = -1;
  195. }
  196. #endif
  197. evsig_init_(base);
  198. return (epollop);
  199. }
  200. static const char *
  201. change_to_string(int change)
  202. {
  203. change &= (EV_CHANGE_ADD|EV_CHANGE_DEL);
  204. if (change == EV_CHANGE_ADD) {
  205. return "add";
  206. } else if (change == EV_CHANGE_DEL) {
  207. return "del";
  208. } else if (change == 0) {
  209. return "none";
  210. } else {
  211. return "???";
  212. }
  213. }
  214. static const char *
  215. epoll_op_to_string(int op)
  216. {
  217. return op == EPOLL_CTL_ADD?"ADD":
  218. op == EPOLL_CTL_DEL?"DEL":
  219. op == EPOLL_CTL_MOD?"MOD":
  220. "???";
  221. }
  222. #define PRINT_CHANGES(op, events, ch, status) \
  223. "Epoll %s(%d) on fd %d " status ". " \
  224. "Old events were %d; " \
  225. "read change was %d (%s); " \
  226. "write change was %d (%s); " \
  227. "close change was %d (%s)", \
  228. epoll_op_to_string(op), \
  229. events, \
  230. ch->fd, \
  231. ch->old_events, \
  232. ch->read_change, \
  233. change_to_string(ch->read_change), \
  234. ch->write_change, \
  235. change_to_string(ch->write_change), \
  236. ch->close_change, \
  237. change_to_string(ch->close_change)
  238. static int
  239. epoll_apply_one_change(struct event_base *base,
  240. struct epollop *epollop,
  241. const struct event_change *ch)
  242. {
  243. struct epoll_event epev;
  244. int op, events = 0;
  245. int idx;
  246. idx = EPOLL_OP_TABLE_INDEX(ch);
  247. op = epoll_op_table[idx].op;
  248. events = epoll_op_table[idx].events;
  249. if (!events) {
  250. EVUTIL_ASSERT(op == 0);
  251. return 0;
  252. }
  253. if ((ch->read_change|ch->write_change|ch->close_change) & EV_CHANGE_ET)
  254. events |= EPOLLET;
  255. memset(&epev, 0, sizeof(epev));
  256. epev.data.fd = ch->fd;
  257. epev.events = events;
  258. if (epoll_ctl(epollop->epfd, op, ch->fd, &epev) == 0) {
  259. event_debug((PRINT_CHANGES(op, epev.events, ch, "okay")));
  260. return 0;
  261. }
  262. switch (op) {
  263. case EPOLL_CTL_MOD:
  264. if (errno == ENOENT) {
  265. /* If a MOD operation fails with ENOENT, the
  266. * fd was probably closed and re-opened. We
  267. * should retry the operation as an ADD.
  268. */
  269. if (epoll_ctl(epollop->epfd, EPOLL_CTL_ADD, ch->fd, &epev) == -1) {
  270. event_warn("Epoll MOD(%d) on %d retried as ADD; that failed too",
  271. (int)epev.events, ch->fd);
  272. return -1;
  273. } else {
  274. event_debug(("Epoll MOD(%d) on %d retried as ADD; succeeded.",
  275. (int)epev.events,
  276. ch->fd));
  277. return 0;
  278. }
  279. }
  280. break;
  281. case EPOLL_CTL_ADD:
  282. if (errno == EEXIST) {
  283. /* If an ADD operation fails with EEXIST,
  284. * either the operation was redundant (as with a
  285. * precautionary add), or we ran into a fun
  286. * kernel bug where using dup*() to duplicate the
  287. * same file into the same fd gives you the same epitem
  288. * rather than a fresh one. For the second case,
  289. * we must retry with MOD. */
  290. if (epoll_ctl(epollop->epfd, EPOLL_CTL_MOD, ch->fd, &epev) == -1) {
  291. event_warn("Epoll ADD(%d) on %d retried as MOD; that failed too",
  292. (int)epev.events, ch->fd);
  293. return -1;
  294. } else {
  295. event_debug(("Epoll ADD(%d) on %d retried as MOD; succeeded.",
  296. (int)epev.events,
  297. ch->fd));
  298. return 0;
  299. }
  300. }
  301. break;
  302. case EPOLL_CTL_DEL:
  303. if (errno == ENOENT || errno == EBADF || errno == EPERM) {
  304. /* If a delete fails with one of these errors,
  305. * that's fine too: we closed the fd before we
  306. * got around to calling epoll_dispatch. */
  307. event_debug(("Epoll DEL(%d) on fd %d gave %s: DEL was unnecessary.",
  308. (int)epev.events,
  309. ch->fd,
  310. strerror(errno)));
  311. return 0;
  312. }
  313. break;
  314. default:
  315. break;
  316. }
  317. event_warn(PRINT_CHANGES(op, epev.events, ch, "failed"));
  318. return -1;
  319. }
  320. static int
  321. epoll_apply_changes(struct event_base *base)
  322. {
  323. struct event_changelist *changelist = &base->changelist;
  324. struct epollop *epollop = base->evbase;
  325. struct event_change *ch;
  326. int r = 0;
  327. int i;
  328. for (i = 0; i < changelist->n_changes; ++i) {
  329. ch = &changelist->changes[i];
  330. if (epoll_apply_one_change(base, epollop, ch) < 0)
  331. r = -1;
  332. }
  333. return (r);
  334. }
  335. static int
  336. epoll_nochangelist_add(struct event_base *base, evutil_socket_t fd,
  337. short old, short events, void *p)
  338. {
  339. struct event_change ch;
  340. ch.fd = fd;
  341. ch.old_events = old;
  342. ch.read_change = ch.write_change = ch.close_change = 0;
  343. if (events & EV_WRITE)
  344. ch.write_change = EV_CHANGE_ADD |
  345. (events & EV_ET);
  346. if (events & EV_READ)
  347. ch.read_change = EV_CHANGE_ADD |
  348. (events & EV_ET);
  349. if (events & EV_CLOSED)
  350. ch.close_change = EV_CHANGE_ADD |
  351. (events & EV_ET);
  352. return epoll_apply_one_change(base, base->evbase, &ch);
  353. }
  354. static int
  355. epoll_nochangelist_del(struct event_base *base, evutil_socket_t fd,
  356. short old, short events, void *p)
  357. {
  358. struct event_change ch;
  359. ch.fd = fd;
  360. ch.old_events = old;
  361. ch.read_change = ch.write_change = ch.close_change = 0;
  362. if (events & EV_WRITE)
  363. ch.write_change = EV_CHANGE_DEL |
  364. (events & EV_ET);
  365. if (events & EV_READ)
  366. ch.read_change = EV_CHANGE_DEL |
  367. (events & EV_ET);
  368. if (events & EV_CLOSED)
  369. ch.close_change = EV_CHANGE_DEL |
  370. (events & EV_ET);
  371. return epoll_apply_one_change(base, base->evbase, &ch);
  372. }
  373. static int
  374. epoll_dispatch(struct event_base *base, struct timeval *tv)
  375. {
  376. struct epollop *epollop = base->evbase;
  377. struct epoll_event *events = epollop->events;
  378. int i, res;
  379. long timeout = -1;
  380. #ifdef USING_TIMERFD
  381. if (epollop->timerfd >= 0) {
  382. struct itimerspec is;
  383. is.it_interval.tv_sec = 0;
  384. is.it_interval.tv_nsec = 0;
  385. if (tv == NULL) {
  386. /* No timeout; disarm the timer. */
  387. is.it_value.tv_sec = 0;
  388. is.it_value.tv_nsec = 0;
  389. } else {
  390. if (tv->tv_sec == 0 && tv->tv_usec == 0) {
  391. /* we need to exit immediately; timerfd can't
  392. * do that. */
  393. timeout = 0;
  394. }
  395. is.it_value.tv_sec = tv->tv_sec;
  396. is.it_value.tv_nsec = tv->tv_usec * 1000;
  397. }
  398. /* TODO: we could avoid unnecessary syscalls here by only
  399. calling timerfd_settime when the top timeout changes, or
  400. when we're called with a different timeval.
  401. */
  402. if (timerfd_settime(epollop->timerfd, 0, &is, NULL) < 0) {
  403. event_warn("timerfd_settime");
  404. }
  405. } else
  406. #endif
  407. if (tv != NULL) {
  408. timeout = evutil_tv_to_msec_(tv);
  409. if (timeout < 0 || timeout > MAX_EPOLL_TIMEOUT_MSEC) {
  410. /* Linux kernels can wait forever if the timeout is
  411. * too big; see comment on MAX_EPOLL_TIMEOUT_MSEC. */
  412. timeout = MAX_EPOLL_TIMEOUT_MSEC;
  413. }
  414. }
  415. epoll_apply_changes(base);
  416. event_changelist_remove_all_(&base->changelist, base);
  417. EVBASE_RELEASE_LOCK(base, th_base_lock);
  418. res = epoll_wait(epollop->epfd, events, epollop->nevents, timeout);
  419. EVBASE_ACQUIRE_LOCK(base, th_base_lock);
  420. if (res == -1) {
  421. if (errno != EINTR) {
  422. event_warn("epoll_wait");
  423. return (-1);
  424. }
  425. return (0);
  426. }
  427. event_debug(("%s: epoll_wait reports %d", __func__, res));
  428. EVUTIL_ASSERT(res <= epollop->nevents);
  429. for (i = 0; i < res; i++) {
  430. int what = events[i].events;
  431. short ev = 0;
  432. #ifdef USING_TIMERFD
  433. if (events[i].data.fd == epollop->timerfd)
  434. continue;
  435. #endif
  436. if (what & EPOLLERR) {
  437. ev = EV_READ | EV_WRITE;
  438. } else if ((what & EPOLLHUP) && !(what & EPOLLRDHUP)) {
  439. ev = EV_READ | EV_WRITE;
  440. } else {
  441. if (what & EPOLLIN)
  442. ev |= EV_READ;
  443. if (what & EPOLLOUT)
  444. ev |= EV_WRITE;
  445. if (what & EPOLLRDHUP)
  446. ev |= EV_CLOSED;
  447. }
  448. if (!ev)
  449. continue;
  450. evmap_io_active_(base, events[i].data.fd, ev | EV_ET);
  451. }
  452. if (res == epollop->nevents && epollop->nevents < MAX_NEVENT) {
  453. /* We used all of the event space this time. We should
  454. be ready for more events next time. */
  455. int new_nevents = epollop->nevents * 2;
  456. struct epoll_event *new_events;
  457. new_events = mm_realloc(epollop->events,
  458. new_nevents * sizeof(struct epoll_event));
  459. if (new_events) {
  460. epollop->events = new_events;
  461. epollop->nevents = new_nevents;
  462. }
  463. }
  464. return (0);
  465. }
  466. static void
  467. epoll_dealloc(struct event_base *base)
  468. {
  469. struct epollop *epollop = base->evbase;
  470. evsig_dealloc_(base);
  471. if (epollop->events)
  472. mm_free(epollop->events);
  473. if (epollop->epfd >= 0)
  474. close(epollop->epfd);
  475. #ifdef USING_TIMERFD
  476. if (epollop->timerfd >= 0)
  477. close(epollop->timerfd);
  478. #endif
  479. memset(epollop, 0, sizeof(struct epollop));
  480. mm_free(epollop);
  481. }
  482. #endif /* EVENT__HAVE_EPOLL */