word_splitter.cpp

Go to the documentation of this file.

// Jubatus: Online machine learning framework for distributed environment
// Copyright (C) 2014 Preferred Networks and Nippon Telegraph and Telephone Corporation.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License version 2.1 as published by the Free Software Foundation.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

#include "word_splitter.hpp"

#include <string>
#include <utility>
#include <vector>

namespace jubatus {
namespace core {
namespace fv_converter {

void word_splitter::extract(
    const std::string& text,
    std::vector<string_feature_element>& result) const {
  std::vector<std::pair<size_t, size_t> > boundaries;
  split(text, boundaries);

  std::vector<string_feature_element> elements;
  for (size_t i = 0; i < boundaries.size(); ++i) {
    size_t begin = boundaries[i].first;
    size_t length = boundaries[i].second;
    elements.push_back(
        string_feature_element(
            begin,
            length,
            text.substr(begin, length),
            1.0));
  }
  elements.swap(result);
}

}  // namespace fv_converter
}  // namespace core
}  // namespace jubatus