Safe to delete analytics documents?



  • I have a Sandbox MongoDB and I'm almost reaching the 512MB limit. The objects collection currently has 291MB according to mLab's dashboard, but when I run my code to calculate the size of all documents grouped by keys, I get 97MB:

    TotalSize 97127973
    [ { size: 27893209, key: 'ip:recent' },
      { size: 3589893, key: 'analytics:uniquevisitors' },
      { size: 3397240, key: 'analytics:pageviews' },
      { size: 2156448, key: 'analytics:pageviews:byCid:2' },
      { size: 2076103, key: 'analytics:errors:404' },
      { size: 2006251, key: 'analytics:pageviews:byCid:11' },
      { size: 1570818, key: 'analytics:pageviews:byCid:13' },
      { size: 1477344, key: 'analytics:pageviews:byCid:5' },
      { size: 1361395, key: 'analytics:pageviews:byCid:15' },
      { size: 1113792, key: 'analytics:pageviews:byCid:1' },
      { size: 1050995, key: 'analytics:pageviews:byCid:16' },
      { size: 1035648, key: 'analytics:pageviews:byCid:6' },
      { size: 1022688, key: 'analytics:pageviews:byCid:3' },
      { size: 844767, key: 'analytics:pageviews:registered' },
      { size: 806802, key: 'analytics:pageviews:guest' },
      { size: 752544, key: 'analytics:pageviews:byCid:4' },
      { size: 646116, key: 'analytics:pageviews:bot' },
      { size: 638890, key: 'group:registered-users:members' },
      { size: 624893, key: 'email:sorted' },
      { size: 570543, key: 'username:sorted' },
      { size: 569470, key: 'email:uid' },
      { size: 528288, key: 'users:joindate' },
      { size: 515562, key: 'users:notvalidated' },
      { size: 514863, key: 'users:online' },
      { size: 514790, key: 'users:reputation' },
      { size: 514677, key: 'username:uid' },
      { size: 514671, key: 'userslug:uid' },
      { size: 510899, key: 'analytics:pageviews:byCid:19' },
      { size: 508116, key: 'users:postcount' },
      { size: 470256, key: 'analytics:pageviews:byCid:21' },
      { size: 422241, key: 'analytics:pageviews:byCid:17' },
      { size: 402841, key: 'analytics:pageviews:byCid:22' },
      { size: 296016, key: 'analytics:posts' },
      { size: 265932, key: 'analytics:errors:503' },
      { size: 242494, key: 'posts:pid' },
      { size: 238523, key: 'analytics:pageviews:byCid:20' },
      { size: 135731, key: 'tid:1172:bookmarks' },
      { size: 134130, key: 'analytics:topics' },
      { size: 133489, key: 'uid:1:tids_read' },
      { size: 121778, key: 'tid:615:bookmarks' },
      { size: 117208, key: 'analytics:posts:byCid:2' },
      { size: 90287, key: 'tid:415:bookmarks' },
      { size: 80986, key: 'uid:472:tids_read' },
      { size: 78450, key: 'tid:1206:bookmarks' },
      { size: 69259, key: 'tid:1334:bookmarks' },
      { size: 67261, key: 'tid:540:bookmarks' },
      { size: 64879, key: 'tid:1282:bookmarks' },
      { size: 63157, key: 'tid:1032:bookmarks' },
      { size: 62433, key: 'cid:2:pids' },
      { size: 61880, key: 'uid:453:tids_read' },
      { size: 60052, key: 'uid:2247:tids_read' },
      { size: 55094, key: 'tid:1335:bookmarks' },
      { size: 53289, key: 'analytics:posts:byCid:15' },
      { size: 50034, key: 'analytics:topics:byCid:2' },
      { size: 49931, key: 'topics:recent' },
      { size: 48765, key: 'tid:1576:bookmarks' },
      { size: 48706, key: 'uid:2261:tids_read' },
      { size: 48428, key: 'uid:3478:tids_read' },
      { size: 48344, key: 'topics:tid' },
      { size: 47591, key: 'topics:posts' },
      { size: 47525, key: 'tid:1076:bookmarks' },
      { size: 47058, key: 'analytics:posts:byCid:11' },
      { size: 46948, key: 'topics:views' },
      { size: 46177, key: 'tid:1154:bookmarks' },
      { size: 45983, key: 'uid:447:tids_read' },
      { size: 44066, key: 'uid:2622:tids_read' },
      { size: 43551, key: 'tid:1095:bookmarks' },
      { size: 42209, key: 'posts:votes' },
      { size: 41479, key: 'uid:0:posts' },
      { size: 38412, key: 'uid:1914:tids_read' },
      { size: 37808, key: 'uid:484:tids_read' },
      { size: 37254, key: 'uid:4016:tids_read' },
      { size: 36394, key: 'uid:0:tids_read' },
      { size: 34891, key: 'tid:1524:bookmarks' },
      { size: 34785, key: 'uid:1:sessionUUID:sessionId' },
      { size: 34686, key: 'analytics:topics:byCid:15' },
      { size: 34350, key: 'cid:21:pids' },
      { size: 33387, key: 'analytics:posts:byCid:21' },
      { size: 32650, key: 'tid:1442:bookmarks' },
      { size: 32461, key: 'cid:11:pids' },
      { size: 31518, key: 'tid:1541:bookmarks' },
      { size: 30799, key: 'uid:1:posts' },
      { size: 30318, key: 'analytics:posts:byCid:17' },
      { size: 30068, key: 'cid:15:pids' },
      { size: 30035, key: 'tid:1601:bookmarks' },
      { size: 29180, key: 'uid:2005:tids_read' },
      { size: 29128, key: 'events:time:user-delete' },
      { size: 29003, key: 'tid:473:bookmarks' },
      { size: 28260, key: 'cid:5:pids' },
      { size: 28171, key: 'uid:862:tids_read' },
      { size: 27981, key: 'post:1485' },
      { size: 27816, key: 'events:time' },
      { size: 27271, key: 'tid:1525:bookmarks' },
      { size: 27091, key: 'uid:647:tids_read' },
      { size: 26844, key: 'tid:1705:bookmarks' },
      { size: 26842, key: 'tid:1718:bookmarks' },
      { size: 26227, key: 'tid:1546:bookmarks' },
      { size: 25200, key: 'tid:1172:posts:votes' },
      { size: 25073, key: 'uid:1008:tids_read' },
      { size: 24890, key: 'uid:6111:tids_read' },
      ... 77671 more items ]
    

    1 - Is it safe to delete the ip:recent and analytics keys?
    2 - Why does mLab show 291MB while my code finds 97MB? I suppose the size in the disk doesn't have anything to do with the size calculated using the BSON module. Here's the code that I use on the objects collection:

    var bson = require('bson')
    function getLargestDocumentsDB(db, callback) {
        _DB.collection(db).find().toArray(function (err, data) {
            var totalSize = 0
            if (data && data.length) {
                var BSON = new bson()
                data.forEach(function (obj) {
                    var size = BSON.calculateObjectSize(obj)
                    totalSize += size
                })
            }
            callback(totalSize)
        })
    }
    

  • Admin Staff

    It is safe to delete the ip:recent and analytics keys, you will just lose the historical data on the dashboard.

    Your calculations probably doesn't include the index data size. You can check /admin/advanced/database to see those sizes.



  • I had 900k documents in my objects database. After running the following, I now have 300k documents and mLab only shows 131MB:

    db.objects.remove({_key: "ip:recent"})
    db.objects.remove({_key: {$regex: /^analytics*/})

    Thank you so much!


Log in to reply
 

Suggested Topics

| |