+ if [j.get(x) for x in 'man_os', 'pers_os', 'virt_host_os', 'bg_datamod'] == \
+ ['none', 'none', 'none', None]:
+ verbose(1, "Skipping %s because all systems are nones" % f)
+ skipped += 1
+ continue
+
+ if 'remote_addr' in j:
+ nwith_ips += 1
+ ip = j['remote_addr']
+ agent = j.get('user_agent', None)
+ previous_entries = ips.get((ip, agent), [])
+ # Let's see if we catch results seekers -- check for how many
+ # fields are identical
+ if len(previous_entries):
+ diffs = [ndiffer(x, j) for x in previous_entries]
+ if min(diffs) < 2:
+ verbose(1, "Skipping %s because there is a previous entry which differs only in %d fields" % (f, min(diffs),))
+ skipped += 1
+ continue
+ ips[(ip, agent)] = previous_entries + [j]
+