visitor.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532
  1. package server
  2. import (
  3. "fmt"
  4. "heckel.io/ntfy/log"
  5. "heckel.io/ntfy/user"
  6. "net/netip"
  7. "sync"
  8. "time"
  9. "golang.org/x/time/rate"
  10. "heckel.io/ntfy/util"
  11. )
  12. const (
  13. // oneDay is an approximation of a day as a time.Duration
  14. oneDay = 24 * time.Hour
  15. // visitorExpungeAfter defines how long a visitor is active before it is removed from memory. This number
  16. // has to be very high to prevent e-mail abuse, but it doesn't really affect the other limits anyway, since
  17. // they are replenished faster (typically).
  18. visitorExpungeAfter = oneDay
  19. // visitorDefaultReservationsLimit is the amount of topic names a user without a tier is allowed to reserve.
  20. // This number is zero, and changing it may have unintended consequences in the web app, or otherwise
  21. visitorDefaultReservationsLimit = int64(0)
  22. // visitorDefaultCallsLimit is the amount of calls a user without a tier is allowed to make.
  23. // This number is zero, because phone numbers have to be verified first.
  24. visitorDefaultCallsLimit = int64(0)
  25. )
  26. // Constants used to convert a tier-user's MessageLimit (see user.Tier) into adequate request limiter
  27. // values (token bucket). This is only used to increase the values in server.yml, never decrease them.
  28. //
  29. // Example: Assuming a user.Tier's MessageLimit is 10,000:
  30. // - the allowed burst is 500 (= 10,000 * 5%), which is < 1000 (the max)
  31. // - the replenish rate is 2 * 10,000 / 24 hours
  32. const (
  33. visitorMessageToRequestLimitBurstRate = 0.05
  34. visitorMessageToRequestLimitBurstMax = 1000
  35. visitorMessageToRequestLimitReplenishFactor = 2
  36. )
  37. // Constants used to convert a tier-user's EmailLimit (see user.Tier) into adequate email limiter
  38. // values (token bucket). Example: Assuming a user.Tier's EmailLimit is 200, the allowed burst is
  39. // 40 (= 200 * 20%), which is <150 (the max).
  40. const (
  41. visitorEmailLimitBurstRate = 0.2
  42. visitorEmailLimitBurstMax = 150
  43. )
  44. // visitor represents an API user, and its associated rate.Limiter used for rate limiting
  45. type visitor struct {
  46. config *Config
  47. messageCache *messageCache
  48. userManager *user.Manager // May be nil
  49. ip netip.Addr // Visitor IP address
  50. user *user.User // Only set if authenticated user, otherwise nil
  51. requestLimiter *rate.Limiter // Rate limiter for (almost) all requests (including messages)
  52. messagesLimiter *util.FixedLimiter // Rate limiter for messages
  53. emailsLimiter *util.RateLimiter // Rate limiter for emails
  54. callsLimiter *util.FixedLimiter // Rate limiter for calls
  55. subscriptionLimiter *util.FixedLimiter // Fixed limiter for active subscriptions (ongoing connections)
  56. bandwidthLimiter *util.RateLimiter // Limiter for attachment bandwidth downloads
  57. accountLimiter *rate.Limiter // Rate limiter for account creation, may be nil
  58. authLimiter *rate.Limiter // Limiter for incorrect login attempts, may be nil
  59. firebase time.Time // Next allowed Firebase message
  60. seen time.Time // Last seen time of this visitor (needed for removal of stale visitors)
  61. mu sync.RWMutex
  62. }
  63. type visitorInfo struct {
  64. Limits *visitorLimits
  65. Stats *visitorStats
  66. }
  67. type visitorLimits struct {
  68. Basis visitorLimitBasis
  69. RequestLimitBurst int
  70. RequestLimitReplenish rate.Limit
  71. MessageLimit int64
  72. MessageExpiryDuration time.Duration
  73. EmailLimit int64
  74. EmailLimitBurst int
  75. EmailLimitReplenish rate.Limit
  76. CallLimit int64
  77. ReservationsLimit int64
  78. AttachmentTotalSizeLimit int64
  79. AttachmentFileSizeLimit int64
  80. AttachmentExpiryDuration time.Duration
  81. AttachmentBandwidthLimit int64
  82. }
  83. type visitorStats struct {
  84. Messages int64
  85. MessagesRemaining int64
  86. Emails int64
  87. EmailsRemaining int64
  88. Calls int64
  89. CallsRemaining int64
  90. Reservations int64
  91. ReservationsRemaining int64
  92. AttachmentTotalSize int64
  93. AttachmentTotalSizeRemaining int64
  94. }
  95. // visitorLimitBasis describes how the visitor limits were derived, either from a user's
  96. // IP address (default config), or from its tier
  97. type visitorLimitBasis string
  98. const (
  99. visitorLimitBasisIP = visitorLimitBasis("ip")
  100. visitorLimitBasisTier = visitorLimitBasis("tier")
  101. )
  102. func newVisitor(conf *Config, messageCache *messageCache, userManager *user.Manager, ip netip.Addr, user *user.User) *visitor {
  103. var messages, emails, calls int64
  104. if user != nil {
  105. messages = user.Stats.Messages
  106. emails = user.Stats.Emails
  107. calls = user.Stats.Calls
  108. }
  109. v := &visitor{
  110. config: conf,
  111. messageCache: messageCache,
  112. userManager: userManager, // May be nil
  113. ip: ip,
  114. user: user,
  115. firebase: time.Unix(0, 0),
  116. seen: time.Now(),
  117. subscriptionLimiter: util.NewFixedLimiter(int64(conf.VisitorSubscriptionLimit)),
  118. requestLimiter: nil, // Set in resetLimiters
  119. messagesLimiter: nil, // Set in resetLimiters, may be nil
  120. emailsLimiter: nil, // Set in resetLimiters
  121. callsLimiter: nil, // Set in resetLimiters, may be nil
  122. bandwidthLimiter: nil, // Set in resetLimiters
  123. accountLimiter: nil, // Set in resetLimiters, may be nil
  124. authLimiter: nil, // Set in resetLimiters, may be nil
  125. }
  126. v.resetLimitersNoLock(messages, emails, calls, false)
  127. return v
  128. }
  129. func (v *visitor) Context() log.Context {
  130. v.mu.RLock()
  131. defer v.mu.RUnlock()
  132. return v.contextNoLock()
  133. }
  134. func (v *visitor) contextNoLock() log.Context {
  135. info := v.infoLightNoLock()
  136. fields := log.Context{
  137. "visitor_id": visitorID(v.ip, v.user),
  138. "visitor_ip": v.ip.String(),
  139. "visitor_seen": util.FormatTime(v.seen),
  140. "visitor_messages": info.Stats.Messages,
  141. "visitor_messages_limit": info.Limits.MessageLimit,
  142. "visitor_messages_remaining": info.Stats.MessagesRemaining,
  143. "visitor_request_limiter_limit": v.requestLimiter.Limit(),
  144. "visitor_request_limiter_tokens": v.requestLimiter.Tokens(),
  145. }
  146. if v.config.SMTPSenderFrom != "" {
  147. fields["visitor_emails"] = info.Stats.Emails
  148. fields["visitor_emails_limit"] = info.Limits.EmailLimit
  149. fields["visitor_emails_remaining"] = info.Stats.EmailsRemaining
  150. }
  151. if v.config.TwilioAccount != "" {
  152. fields["visitor_calls"] = info.Stats.Calls
  153. fields["visitor_calls_limit"] = info.Limits.CallLimit
  154. fields["visitor_calls_remaining"] = info.Stats.CallsRemaining
  155. }
  156. if v.authLimiter != nil {
  157. fields["visitor_auth_limiter_limit"] = v.authLimiter.Limit()
  158. fields["visitor_auth_limiter_tokens"] = v.authLimiter.Tokens()
  159. }
  160. if v.user != nil {
  161. fields["user_id"] = v.user.ID
  162. fields["user_name"] = v.user.Name
  163. if v.user.Tier != nil {
  164. for field, value := range v.user.Tier.Context() {
  165. fields[field] = value
  166. }
  167. }
  168. if v.user.Billing.StripeCustomerID != "" {
  169. fields["stripe_customer_id"] = v.user.Billing.StripeCustomerID
  170. }
  171. if v.user.Billing.StripeSubscriptionID != "" {
  172. fields["stripe_subscription_id"] = v.user.Billing.StripeSubscriptionID
  173. }
  174. }
  175. return fields
  176. }
  177. func visitorExtendedInfoContext(info *visitorInfo) log.Context {
  178. return log.Context{
  179. "visitor_reservations": info.Stats.Reservations,
  180. "visitor_reservations_limit": info.Limits.ReservationsLimit,
  181. "visitor_reservations_remaining": info.Stats.ReservationsRemaining,
  182. "visitor_attachment_total_size": info.Stats.AttachmentTotalSize,
  183. "visitor_attachment_total_size_limit": info.Limits.AttachmentTotalSizeLimit,
  184. "visitor_attachment_total_size_remaining": info.Stats.AttachmentTotalSizeRemaining,
  185. }
  186. }
  187. func (v *visitor) RequestAllowed() bool {
  188. v.mu.RLock() // limiters could be replaced!
  189. defer v.mu.RUnlock()
  190. return v.requestLimiter.Allow()
  191. }
  192. func (v *visitor) FirebaseAllowed() bool {
  193. v.mu.RLock()
  194. defer v.mu.RUnlock()
  195. return !time.Now().Before(v.firebase)
  196. }
  197. func (v *visitor) FirebaseTemporarilyDeny() {
  198. v.mu.Lock()
  199. defer v.mu.Unlock()
  200. v.firebase = time.Now().Add(v.config.FirebaseQuotaExceededPenaltyDuration)
  201. }
  202. func (v *visitor) MessageAllowed() bool {
  203. v.mu.RLock() // limiters could be replaced!
  204. defer v.mu.RUnlock()
  205. return v.messagesLimiter.Allow()
  206. }
  207. func (v *visitor) EmailAllowed() bool {
  208. v.mu.RLock() // limiters could be replaced!
  209. defer v.mu.RUnlock()
  210. return v.emailsLimiter.Allow()
  211. }
  212. func (v *visitor) CallAllowed() bool {
  213. v.mu.RLock() // limiters could be replaced!
  214. defer v.mu.RUnlock()
  215. return v.callsLimiter.Allow()
  216. }
  217. func (v *visitor) SubscriptionAllowed() bool {
  218. v.mu.RLock() // limiters could be replaced!
  219. defer v.mu.RUnlock()
  220. return v.subscriptionLimiter.Allow()
  221. }
  222. // AuthAllowed returns true if an auth request can be attempted (> 1 token available)
  223. func (v *visitor) AuthAllowed() bool {
  224. v.mu.RLock() // limiters could be replaced!
  225. defer v.mu.RUnlock()
  226. if v.authLimiter == nil {
  227. return true
  228. }
  229. return v.authLimiter.Tokens() > 1
  230. }
  231. // AuthFailed records an auth failure
  232. func (v *visitor) AuthFailed() {
  233. v.mu.RLock() // limiters could be replaced!
  234. defer v.mu.RUnlock()
  235. if v.authLimiter != nil {
  236. v.authLimiter.Allow()
  237. }
  238. }
  239. // AccountCreationAllowed returns true if a new account can be created
  240. func (v *visitor) AccountCreationAllowed() bool {
  241. v.mu.RLock() // limiters could be replaced!
  242. defer v.mu.RUnlock()
  243. if v.accountLimiter == nil || (v.accountLimiter != nil && v.accountLimiter.Tokens() < 1) {
  244. return false
  245. }
  246. return true
  247. }
  248. // AccountCreated decreases the account limiter. This is to be called after an account was created.
  249. func (v *visitor) AccountCreated() {
  250. v.mu.RLock() // limiters could be replaced!
  251. defer v.mu.RUnlock()
  252. if v.accountLimiter != nil {
  253. v.accountLimiter.Allow()
  254. }
  255. }
  256. func (v *visitor) BandwidthAllowed(bytes int64) bool {
  257. v.mu.RLock() // limiters could be replaced!
  258. defer v.mu.RUnlock()
  259. return v.bandwidthLimiter.AllowN(bytes)
  260. }
  261. func (v *visitor) RemoveSubscription() {
  262. v.mu.RLock()
  263. defer v.mu.RUnlock()
  264. v.subscriptionLimiter.AllowN(-1)
  265. }
  266. func (v *visitor) Keepalive() {
  267. v.mu.Lock()
  268. defer v.mu.Unlock()
  269. v.seen = time.Now()
  270. }
  271. func (v *visitor) BandwidthLimiter() util.Limiter {
  272. v.mu.RLock() // limiters could be replaced!
  273. defer v.mu.RUnlock()
  274. return v.bandwidthLimiter
  275. }
  276. func (v *visitor) Stale() bool {
  277. v.mu.RLock()
  278. defer v.mu.RUnlock()
  279. return time.Since(v.seen) > visitorExpungeAfter
  280. }
  281. func (v *visitor) Stats() *user.Stats {
  282. v.mu.RLock() // limiters could be replaced!
  283. defer v.mu.RUnlock()
  284. return &user.Stats{
  285. Messages: v.messagesLimiter.Value(),
  286. Emails: v.emailsLimiter.Value(),
  287. Calls: v.callsLimiter.Value(),
  288. }
  289. }
  290. func (v *visitor) ResetStats() {
  291. v.mu.RLock() // limiters could be replaced!
  292. defer v.mu.RUnlock()
  293. v.emailsLimiter.Reset()
  294. v.messagesLimiter.Reset()
  295. v.callsLimiter.Reset()
  296. }
  297. // User returns the visitor user, or nil if there is none
  298. func (v *visitor) User() *user.User {
  299. v.mu.RLock()
  300. defer v.mu.RUnlock()
  301. return v.user // May be nil
  302. }
  303. // IP returns the visitor IP address
  304. func (v *visitor) IP() netip.Addr {
  305. v.mu.RLock()
  306. defer v.mu.RUnlock()
  307. return v.ip
  308. }
  309. // Authenticated returns true if a user successfully authenticated
  310. func (v *visitor) Authenticated() bool {
  311. v.mu.RLock()
  312. defer v.mu.RUnlock()
  313. return v.user != nil
  314. }
  315. // SetUser sets the visitors user to the given value
  316. func (v *visitor) SetUser(u *user.User) {
  317. v.mu.Lock()
  318. defer v.mu.Unlock()
  319. shouldResetLimiters := v.user.TierID() != u.TierID() // TierID works with nil receiver
  320. v.user = u // u may be nil!
  321. if shouldResetLimiters {
  322. var messages, emails, calls int64
  323. if u != nil {
  324. messages, emails, calls = u.Stats.Messages, u.Stats.Emails, u.Stats.Calls
  325. }
  326. v.resetLimitersNoLock(messages, emails, calls, true)
  327. }
  328. }
  329. // MaybeUserID returns the user ID of the visitor (if any). If this is an anonymous visitor,
  330. // an empty string is returned.
  331. func (v *visitor) MaybeUserID() string {
  332. v.mu.RLock()
  333. defer v.mu.RUnlock()
  334. if v.user != nil {
  335. return v.user.ID
  336. }
  337. return ""
  338. }
  339. func (v *visitor) resetLimitersNoLock(messages, emails, calls int64, enqueueUpdate bool) {
  340. limits := v.limitsNoLock()
  341. v.requestLimiter = rate.NewLimiter(limits.RequestLimitReplenish, limits.RequestLimitBurst)
  342. v.messagesLimiter = util.NewFixedLimiterWithValue(limits.MessageLimit, messages)
  343. v.emailsLimiter = util.NewRateLimiterWithValue(limits.EmailLimitReplenish, limits.EmailLimitBurst, emails)
  344. v.callsLimiter = util.NewFixedLimiterWithValue(limits.CallLimit, calls)
  345. v.bandwidthLimiter = util.NewBytesLimiter(int(limits.AttachmentBandwidthLimit), oneDay)
  346. if v.user == nil {
  347. v.accountLimiter = rate.NewLimiter(rate.Every(v.config.VisitorAccountCreationLimitReplenish), v.config.VisitorAccountCreationLimitBurst)
  348. v.authLimiter = rate.NewLimiter(rate.Every(v.config.VisitorAuthFailureLimitReplenish), v.config.VisitorAuthFailureLimitBurst)
  349. } else {
  350. v.accountLimiter = nil // Users cannot create accounts when logged in
  351. v.authLimiter = nil // Users are already logged in, no need to limit requests
  352. }
  353. if enqueueUpdate && v.user != nil {
  354. go v.userManager.EnqueueUserStats(v.user.ID, &user.Stats{
  355. Messages: messages,
  356. Emails: emails,
  357. Calls: calls,
  358. })
  359. }
  360. log.Fields(v.contextNoLock()).Debug("Rate limiters reset for visitor") // Must be after function, because contextNoLock() describes rate limiters
  361. }
  362. func (v *visitor) Limits() *visitorLimits {
  363. v.mu.RLock()
  364. defer v.mu.RUnlock()
  365. return v.limitsNoLock()
  366. }
  367. func (v *visitor) limitsNoLock() *visitorLimits {
  368. if v.user != nil && v.user.Tier != nil {
  369. return tierBasedVisitorLimits(v.config, v.user.Tier)
  370. }
  371. return configBasedVisitorLimits(v.config)
  372. }
  373. func tierBasedVisitorLimits(conf *Config, tier *user.Tier) *visitorLimits {
  374. return &visitorLimits{
  375. Basis: visitorLimitBasisTier,
  376. RequestLimitBurst: util.MinMax(int(float64(tier.MessageLimit)*visitorMessageToRequestLimitBurstRate), conf.VisitorRequestLimitBurst, visitorMessageToRequestLimitBurstMax),
  377. RequestLimitReplenish: util.Max(rate.Every(conf.VisitorRequestLimitReplenish), dailyLimitToRate(tier.MessageLimit*visitorMessageToRequestLimitReplenishFactor)),
  378. MessageLimit: tier.MessageLimit,
  379. MessageExpiryDuration: tier.MessageExpiryDuration,
  380. EmailLimit: tier.EmailLimit,
  381. EmailLimitBurst: util.MinMax(int(float64(tier.EmailLimit)*visitorEmailLimitBurstRate), conf.VisitorEmailLimitBurst, visitorEmailLimitBurstMax),
  382. EmailLimitReplenish: dailyLimitToRate(tier.EmailLimit),
  383. CallLimit: tier.CallLimit,
  384. ReservationsLimit: tier.ReservationLimit,
  385. AttachmentTotalSizeLimit: tier.AttachmentTotalSizeLimit,
  386. AttachmentFileSizeLimit: tier.AttachmentFileSizeLimit,
  387. AttachmentExpiryDuration: tier.AttachmentExpiryDuration,
  388. AttachmentBandwidthLimit: tier.AttachmentBandwidthLimit,
  389. }
  390. }
  391. func configBasedVisitorLimits(conf *Config) *visitorLimits {
  392. messagesLimit := replenishDurationToDailyLimit(conf.VisitorRequestLimitReplenish) // Approximation!
  393. if conf.VisitorMessageDailyLimit > 0 {
  394. messagesLimit = int64(conf.VisitorMessageDailyLimit)
  395. }
  396. return &visitorLimits{
  397. Basis: visitorLimitBasisIP,
  398. RequestLimitBurst: conf.VisitorRequestLimitBurst,
  399. RequestLimitReplenish: rate.Every(conf.VisitorRequestLimitReplenish),
  400. MessageLimit: messagesLimit,
  401. MessageExpiryDuration: conf.CacheDuration,
  402. EmailLimit: replenishDurationToDailyLimit(conf.VisitorEmailLimitReplenish), // Approximation!
  403. EmailLimitBurst: conf.VisitorEmailLimitBurst,
  404. EmailLimitReplenish: rate.Every(conf.VisitorEmailLimitReplenish),
  405. CallLimit: visitorDefaultCallsLimit,
  406. ReservationsLimit: visitorDefaultReservationsLimit,
  407. AttachmentTotalSizeLimit: conf.VisitorAttachmentTotalSizeLimit,
  408. AttachmentFileSizeLimit: conf.AttachmentFileSizeLimit,
  409. AttachmentExpiryDuration: conf.AttachmentExpiryDuration,
  410. AttachmentBandwidthLimit: conf.VisitorAttachmentDailyBandwidthLimit,
  411. }
  412. }
  413. func (v *visitor) Info() (*visitorInfo, error) {
  414. v.mu.RLock()
  415. info := v.infoLightNoLock()
  416. v.mu.RUnlock()
  417. // Attachment stats from database
  418. var attachmentsBytesUsed int64
  419. var err error
  420. u := v.User()
  421. if u != nil {
  422. attachmentsBytesUsed, err = v.messageCache.AttachmentBytesUsedByUser(u.ID)
  423. } else {
  424. attachmentsBytesUsed, err = v.messageCache.AttachmentBytesUsedBySender(v.IP().String())
  425. }
  426. if err != nil {
  427. return nil, err
  428. }
  429. info.Stats.AttachmentTotalSize = attachmentsBytesUsed
  430. info.Stats.AttachmentTotalSizeRemaining = zeroIfNegative(info.Limits.AttachmentTotalSizeLimit - attachmentsBytesUsed)
  431. // Reservation stats from database
  432. var reservations int64
  433. if v.userManager != nil && u != nil {
  434. reservations, err = v.userManager.ReservationsCount(u.Name)
  435. if err != nil {
  436. return nil, err
  437. }
  438. }
  439. info.Stats.Reservations = reservations
  440. info.Stats.ReservationsRemaining = zeroIfNegative(info.Limits.ReservationsLimit - reservations)
  441. return info, nil
  442. }
  443. func (v *visitor) infoLightNoLock() *visitorInfo {
  444. messages := v.messagesLimiter.Value()
  445. emails := v.emailsLimiter.Value()
  446. calls := v.callsLimiter.Value()
  447. limits := v.limitsNoLock()
  448. stats := &visitorStats{
  449. Messages: messages,
  450. MessagesRemaining: zeroIfNegative(limits.MessageLimit - messages),
  451. Emails: emails,
  452. EmailsRemaining: zeroIfNegative(limits.EmailLimit - emails),
  453. Calls: calls,
  454. CallsRemaining: zeroIfNegative(limits.CallLimit - calls),
  455. }
  456. return &visitorInfo{
  457. Limits: limits,
  458. Stats: stats,
  459. }
  460. }
  461. func zeroIfNegative(value int64) int64 {
  462. if value < 0 {
  463. return 0
  464. }
  465. return value
  466. }
  467. func replenishDurationToDailyLimit(duration time.Duration) int64 {
  468. return int64(oneDay / duration)
  469. }
  470. func dailyLimitToRate(limit int64) rate.Limit {
  471. return rate.Limit(limit) * rate.Every(oneDay)
  472. }
  473. func visitorID(ip netip.Addr, u *user.User) string {
  474. if u != nil && u.Tier != nil {
  475. return fmt.Sprintf("user:%s", u.ID)
  476. }
  477. return fmt.Sprintf("ip:%s", ip.String())
  478. }