#include <lof_storage.hpp>

Collaboration diagram for jubatus::core::anomaly::lof_storage:

Classes
struct	config

Public Member Functions
void	clear ()

float	collect_lrds (const common::sfv_t &query, jubatus::util::data::unordered_map< std::string, float > &neighbor_lrd) const

float	collect_lrds (const std::string &id, jubatus::util::data::unordered_map< std::string, float > &neighbor_lrd) const

void	get_all_row_ids (std::vector< std::string > &ids) const

void	get_diff (lof_table_t &diff) const

float	get_kdist (const std::string &row) const

float	get_lrd (const std::string &row) const

storage::version	get_version () const

bool	has_row (const std::string &row) const

	lof_storage ()

	lof_storage (jubatus::util::lang::shared_ptr< core::recommender::recommender_base > nn_engine)

	lof_storage (const config &config, jubatus::util::lang::shared_ptr< core::recommender::recommender_base > nn_engine)

void	mix (const lof_table_t &lhs, lof_table_t &rhs) const

	MSGPACK_DEFINE (lof_table_, lof_table_diff_, neighbor_num_, reverse_nn_num_)

std::string	name () const

void	pack (framework::packer &packer) const

bool	put_diff (const lof_table_t &mixed_diff)

void	remove_row (const std::string &row)

void	set_nn_engine (jubatus::util::lang::shared_ptr< core::recommender::recommender_base > nn_engine)

void	unpack (msgpack::object o)

void	update_all ()

void	update_row (const std::string &row, const common::sfv_t &diff)

virtual	~lof_storage ()

Static Public Attributes
static const uint32_t	DEFAULT_NEIGHBOR_NUM = 10

static const uint32_t	DEFAULT_REVERSE_NN_NUM = 30

Private Member Functions
float	collect_lrds_from_neighbors (const std::vector< std::pair< std::string, float > > &neighbors, jubatus::util::data::unordered_map< std::string, float > &neighbor_lrd) const

void	collect_neighbors (const std::string &row, jubatus::util::data::unordered_set< std::string > &nn) const

void	update_entries (const jubatus::util::data::unordered_set< std::string > &rows)

void	update_kdist (const std::string &row)

void	update_kdist_with_neighbors (const std::string &row, const std::vector< std::pair< std::string, float > > &neighbors)

void	update_lrd (const std::string &row)

void	update_lrd_with_neighbors (const std::string &row, const std::vector< std::pair< std::string, float > > &neighbors)

Static Private Member Functions
static bool	is_removed (const lof_entry &entry)

static void	mark_removed (lof_entry &entry)

Private Attributes
lof_table_t	lof_table_

lof_table_t	lof_table_diff_

uint32_t	neighbor_num_

jubatus::util::lang::shared_ptr< core::recommender::recommender_base >	nn_engine_

uint32_t	reverse_nn_num_

Detailed Description

Definition at line 52 of file lof_storage.hpp.

Constructor & Destructor Documentation

jubatus::core::anomaly::lof_storage::lof_storage ( )

Definition at line 59 of file lof_storage.cpp.

     : neighbor_num_(DEFAULT_NEIGHBOR_NUM),
       reverse_nn_num_(DEFAULT_REVERSE_NN_NUM),
       nn_engine_(recommender::recommender_factory::create_recommender(
           "euclid_lsh",
           common::jsonconfig::config(jubatus::util::text::json::to_json(
               recommender::euclid_lsh::config())), "")) {
 }

jubatus::core::anomaly::lof_storage::lof_storage ( jubatus::util::lang::shared_ptr< core::recommender::recommender_base > nn_engine )

explicit

jubatus::core::anomaly::lof_storage::lof_storage	(	const config &	config,
		jubatus::util::lang::shared_ptr< core::recommender::recommender_base >	nn_engine
	)

explicit

jubatus::core::anomaly::lof_storage::~lof_storage ( )

virtual

Definition at line 83 of file lof_storage.cpp.

83 {

84 }

Member Function Documentation

void jubatus::core::anomaly::lof_storage::clear ( )

Definition at line 125 of file lof_storage.cpp.

References lof_table_, lof_table_diff_, and nn_engine_.

                         {
   lof_table_t().swap(lof_table_);
   lof_table_t().swap(lof_table_diff_);
   nn_engine_->clear();
 }

float jubatus::core::anomaly::lof_storage::collect_lrds	(	const common::sfv_t &	query,
		jubatus::util::data::unordered_map< std::string, float > &	neighbor_lrd
	)		const

float jubatus::core::anomaly::lof_storage::collect_lrds	(	const std::string &	id,
		jubatus::util::data::unordered_map< std::string, float > &	neighbor_lrd
	)		const

float jubatus::core::anomaly::lof_storage::collect_lrds_from_neighbors	(	const std::vector< std::pair< std::string, float > > &	neighbors,
		jubatus::util::data::unordered_map< std::string, float > &	neighbor_lrd
	)		const

private

Based on the given neighbors (list of ID and the distance to it), get the LRDs for each neighbor points and return the calculated LRD for the point. neighbors donesn't have to be sorted.

Definition at line 260 of file lof_storage.cpp.

References get_kdist(), and get_lrd().

                                                       {
   if (neighbors.empty()) {
     return numeric_limits<float>::infinity();
   }
 
   // collect lrd values of the nearest neighbors
   neighbor_lrd.clear();
   for (size_t i = 0; i < neighbors.size(); ++i) {
     neighbor_lrd[neighbors[i].first] = get_lrd(neighbors[i].first);
   }
 
   // return lrd of the query
   float sum_reachability = 0;
   for (size_t i = 0; i < neighbors.size(); ++i) {
     sum_reachability += max(neighbors[i].second, get_kdist(neighbors[i].first));
   }
 
   // All the neighbors seem to have a same feature vector.
   if (sum_reachability == 0) {
     return numeric_limits<float>::infinity();
   }
 
   return neighbors.size() / sum_reachability;
 }

Here is the call graph for this function:

void jubatus::core::anomaly::lof_storage::collect_neighbors	(	const std::string &	row,
		jubatus::util::data::unordered_set< std::string > &	nn
	)		const

private

Collect neighbors for the given ID.

Definition at line 290 of file lof_storage.cpp.

References nn_engine_, and reverse_nn_num_.

Referenced by update_row().

                                      {
   vector<pair<string, float> > neighbors;
   nn_engine_->neighbor_row(row, neighbors, reverse_nn_num_);
 
   for (size_t i = 0; i < neighbors.size(); ++i) {
     nn.insert(neighbors[i].first);
   }
 }

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::get_all_row_ids ( std::vector< std::string > & ids ) const

Definition at line 131 of file lof_storage.cpp.

References nn_engine_.

Referenced by update_all().

                                                            {
   nn_engine_->get_all_row_ids(ids);
 }

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::get_diff ( lof_table_t & diff ) const

Definition at line 215 of file lof_storage.cpp.

References lof_table_diff_.

                                                   {
   diff = lof_table_diff_;
 }

float jubatus::core::anomaly::lof_storage::get_kdist ( const std::string & row ) const

Definition at line 158 of file lof_storage.cpp.

References is_removed(), JUBATUS_EXCEPTION, lof_table_, and lof_table_diff_.

Referenced by collect_lrds_from_neighbors(), and update_lrd_with_neighbors().

                                                     {
   lof_table_t::const_iterator it = lof_table_diff_.find(row);
   if (it == lof_table_diff_.end()) {
     it = lof_table_.find(row);
     if (it == lof_table_.end()) {
       throw JUBATUS_EXCEPTION(
         common::exception::runtime_error("specified row does not exist")
         << common::exception::error_message("row id: " + row));
     }
   } else if (is_removed(it->second)) {
     throw JUBATUS_EXCEPTION(
       common::exception::runtime_error("specified row is recently removed")
       << common::exception::error_message("row id: " + row));
   }
   return it->second.kdist;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

float jubatus::core::anomaly::lof_storage::get_lrd ( const std::string & row ) const

Definition at line 175 of file lof_storage.cpp.

References is_removed(), JUBATUS_EXCEPTION, lof_table_, and lof_table_diff_.

Referenced by collect_lrds_from_neighbors().

                                                   {
   lof_table_t::const_iterator it = lof_table_diff_.find(row);
   if (it == lof_table_diff_.end()) {
     it = lof_table_.find(row);
     if (it == lof_table_.end()) {
       throw JUBATUS_EXCEPTION(
         common::exception::runtime_error("specified row does not exist")
         << common::exception::error_message("row id: " + row));
     }
   } else if (is_removed(it->second)) {
     throw JUBATUS_EXCEPTION(
       common::exception::runtime_error("specified row is recently removed")
       << common::exception::error_message("row id: " + row));
   }
   return it->second.lrd;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

storage::version jubatus::core::anomaly::lof_storage::get_version ( ) const

inline

Definition at line 120 of file lof_storage.hpp.

                                      {
     return storage::version();
   }

bool jubatus::core::anomaly::lof_storage::has_row ( const std::string & row ) const

Definition at line 192 of file lof_storage.cpp.

References lof_table_, and lof_table_diff_.

                                                  {
   return lof_table_diff_.count(row) > 0 || lof_table_.count(row) > 0;
 }

bool jubatus::core::anomaly::lof_storage::is_removed ( const lof_entry & entry )

staticprivate

Definition at line 250 of file lof_storage.cpp.

References jubatus::core::anomaly::lof_entry::kdist.

Referenced by get_kdist(), get_lrd(), mix(), and put_diff().

                                                    {
   return entry.kdist < 0;
 }

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::mark_removed ( lof_entry & entry )

staticprivate

Definition at line 245 of file lof_storage.cpp.

References jubatus::core::anomaly::lof_entry::kdist.

Referenced by mix(), and remove_row().

                                                {
   entry.kdist = -1;
 }

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::mix	(	const lof_table_t &	lhs,
		lof_table_t &	rhs
	)		const

Definition at line 232 of file lof_storage.cpp.

References is_removed(), and mark_removed().

                                                                     {
   for (lof_table_t::const_iterator it = lhs.begin(); it != lhs.end(); ++it) {
     if (is_removed(it->second)) {
       mark_removed(rhs[it->first]);
     } else {
       rhs.insert(*it);
     }
   }
 }

Here is the call graph for this function:

jubatus::core::anomaly::lof_storage::MSGPACK_DEFINE	(	lof_table_	,
		lof_table_diff_	,
		neighbor_num_	,
		reverse_nn_num_
	)

string jubatus::core::anomaly::lof_storage::name ( ) const

Definition at line 154 of file lof_storage.cpp.

                                {
   return "lof_storage";
 }

void jubatus::core::anomaly::lof_storage::pack ( framework::packer & packer ) const

inline

Definition at line 124 of file lof_storage.hpp.

                                            {
     packer.pack(*this);
   }

bool jubatus::core::anomaly::lof_storage::put_diff ( const lof_table_t & mixed_diff )

Definition at line 219 of file lof_storage.cpp.

References is_removed(), lof_table_, and lof_table_diff_.

                                                         {
   for (lof_table_t::const_iterator it = mixed_diff.begin();
        it != mixed_diff.end(); ++it) {
     if (is_removed(it->second)) {
       lof_table_.erase(it->first);
     } else {
       lof_table_[it->first] = it->second;
     }
   }
   lof_table_diff_.clear();
   return true;
 }

Here is the call graph for this function:

void jubatus::core::anomaly::lof_storage::remove_row ( const std::string & row )

Definition at line 120 of file lof_storage.cpp.

References lof_table_diff_, mark_removed(), and nn_engine_.

                                               {
   mark_removed(lof_table_diff_[row]);
   nn_engine_->clear_row(row);
 }

Here is the call graph for this function:

void jubatus::core::anomaly::lof_storage::set_nn_engine ( jubatus::util::lang::shared_ptr< core::recommender::recommender_base > nn_engine )

Definition at line 210 of file lof_storage.cpp.

References nn_engine_.

                                                        {
   nn_engine_ = nn_engine;
 }

void jubatus::core::anomaly::lof_storage::unpack ( msgpack::object o )

inline

Definition at line 127 of file lof_storage.hpp.

                                {
     o.convert(this);
   }

void jubatus::core::anomaly::lof_storage::update_all ( )

Definition at line 196 of file lof_storage.cpp.

References get_all_row_ids(), update_kdist(), and update_lrd().

                              {
   vector<string> ids;
   get_all_row_ids(ids);
 
   // NOTE: These two loops are separated, since update_lrd requires new kdist
   // values of k-NN.
   for (size_t i = 0; i < ids.size(); ++i) {
     update_kdist(ids[i]);
   }
   for (size_t i = 0; i < ids.size(); ++i) {
     update_lrd(ids[i]);
   }
 }

Here is the call graph for this function:

void jubatus::core::anomaly::lof_storage::update_entries ( const jubatus::util::data::unordered_set< std::string > & rows )

private

Update kdist and LRD for given points and its neighbors.

Definition at line 304 of file lof_storage.cpp.

References neighbor_num_, nn_engine_, update_kdist_with_neighbors(), and update_lrd_with_neighbors().

Referenced by update_row().

                                                                   {
   // NOTE: These two loops are separated, since update_lrd requires new kdist
   // values of k-NN.
   typedef unordered_map<string, vector<pair<string, float> > >
     rows_to_neighbors_type;
 
   rows_to_neighbors_type rows_to_neighbors;
   for (unordered_set<string>::const_iterator it = rows.begin();
        it != rows.end(); ++it) {
     nn_engine_->neighbor_row(*it, rows_to_neighbors[*it], neighbor_num_);
   }
 
   for (rows_to_neighbors_type::const_iterator it = rows_to_neighbors.begin();
        it != rows_to_neighbors.end(); ++it) {
     update_kdist_with_neighbors(it->first, it->second);
   }
   for (rows_to_neighbors_type::const_iterator it = rows_to_neighbors.begin();
        it != rows_to_neighbors.end(); ++it) {
     update_lrd_with_neighbors(it->first, it->second);
   }
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::update_kdist ( const std::string & row )

private

Update kdist for the row.

Definition at line 329 of file lof_storage.cpp.

References neighbor_num_, nn_engine_, and update_kdist_with_neighbors().

Referenced by update_all().

                                                 {
   vector<pair<string, float> > neighbors;
   nn_engine_->neighbor_row(row, neighbors, neighbor_num_);
   update_kdist_with_neighbors(row, neighbors);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::update_kdist_with_neighbors	(	const std::string &	row,
		const std::vector< std::pair< std::string, float > > &	neighbors
	)

private

Update kdist for the row using given NN search result (neighbors). Note that this method expects neighbors to be sorted by score.

Definition at line 339 of file lof_storage.cpp.

References lof_table_diff_.

Referenced by update_entries(), and update_kdist().

                                                    {
   if (!neighbors.empty()) {
     lof_table_diff_[row].kdist = neighbors.back().second;
   }
 }

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::update_lrd ( const std::string & row )

private

Update LRD for the row.

Definition at line 350 of file lof_storage.cpp.

References neighbor_num_, nn_engine_, and update_lrd_with_neighbors().

Referenced by update_all().

                                               {
   vector<pair<string, float> > neighbors;
   nn_engine_->neighbor_row(row, neighbors, neighbor_num_);
   update_lrd_with_neighbors(row, neighbors);
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::update_lrd_with_neighbors	(	const std::string &	row,
		const std::vector< std::pair< std::string, float > > &	neighbors
	)

private

Update LRD for the row using given NN search result (neighbors).

Definition at line 359 of file lof_storage.cpp.

References get_kdist(), lof_table_diff_, and neighbor_num_.

Referenced by update_entries(), and update_lrd().

                                                                       {
   if (neighbors.empty()) {
     lof_table_diff_[row].lrd = 1;
     return;
   }
 
   const size_t length = min(neighbors.size(), size_t(neighbor_num_));
   float sum_reachability = 0;
   for (size_t i = 0; i < length; ++i) {
     sum_reachability += max(neighbors[i].second, get_kdist(neighbors[i].first));
   }
 
   if (sum_reachability == 0) {
     lof_table_diff_[row].lrd = numeric_limits<float>::infinity();
     return;
   }
 
   lof_table_diff_[row].lrd = length / sum_reachability;
 }

Here is the call graph for this function:

Here is the caller graph for this function:

void jubatus::core::anomaly::lof_storage::update_row	(	const std::string &	row,
		const common::sfv_t &	diff
	)

Definition at line 135 of file lof_storage.cpp.

References collect_neighbors(), nn_engine_, and update_entries().

                                                                        {
   unordered_set<string> update_set;
 
   {
     common::sfv_t query;
     nn_engine_->decode_row(row, query);
     if (!query.empty()) {
       collect_neighbors(row, update_set);
     }
   }
 
   nn_engine_->update_row(row, diff);
   collect_neighbors(row, update_set);
 
   update_set.insert(row);
 
   update_entries(update_set);
 }