Changeset 90

Timestamp:

07/09/08 16:02:53 (3 months ago)

Author:

deveiant

Message:

Checkpoint -- conversion of database files and basic rake infrastructure from rake-tasklibs external
working.

Location:

branches/rakefile-work

Files:

• . (modified) (1 prop)
• Rakefile (modified) (4 diffs)
• convertdb.rb (modified) (3 diffs)
• lib/wordnet.rb (modified) (1 diff)
• lib/wordnet/constants.rb (modified) (4 diffs)
• lib/wordnet/synset.rb (modified) (12 diffs)

branches/rakefile-work

@@ -1 @@
-rake svn://deveiate.org/rake-tasklibs/trunks
+rake svn://deveiate.org/rake-tasklibs/trunk

@@ -1 @@
-rake svn://deveiate.org/rake-tasklibs/trunks
+rake svn://deveiate.org/rake-tasklibs/trunk

branches/rakefile-work/Rakefile

@@ -1 +1 @@
 #!rake
 #
-# Ruby-WordNet Rakefile
-#
-# Copyright (c) 2008, The FaerieMUD Consortium
+# WordNet rakefile
+#
+# Based on various other Rakefiles, especially one by Ben Bleything
+#
+# Copyright (c) 2008 The FaerieMUD Consortium
 #
 # Authors:
@@ -12 +14 @@
     require 'pathname'
     basedir = Pathname.new( __FILE__ ).dirname
-    libdir = basedir + 'lib'
+
+    libdir = basedir + "lib"
+    extdir = basedir + "ext"
 
     $LOAD_PATH.unshift( libdir.to_s ) unless $LOAD_PATH.include?( libdir.to_s )
+    $LOAD_PATH.unshift( extdir.to_s ) unless $LOAD_PATH.include?( extdir.to_s )
 }
 
 
+require 'rbconfig'
 require 'rubygems'
 require 'rake'
 require 'rake/rdoctask'
+require 'rake/testtask'
 require 'rake/packagetask'
-require 'rake/gempackagetask'
-require 'pathname'
+require 'rake/clean'
 
 $dryrun = false
 
-# Pathname constants
+### Config constants
 BASEDIR       = Pathname.new( __FILE__ ).dirname.relative_path_from( Pathname.getwd )
 LIBDIR        = BASEDIR + 'lib'
+EXTDIR        = BASEDIR + 'ext'
 DOCSDIR       = BASEDIR + 'docs'
-RAKEDIR       = BASEDIR + 'rake'
-RDOCDIR       = BASEDIR + 'docs'
 PKGDIR        = BASEDIR + 'pkg'
-
 RAKE_TASKDIR  = BASEDIR + 'rake'
-ARTIFACTS_DIR = Pathname.new( ENV['CC_BUILD_ARTIFACTS'] || '' )
-
-TEXT_FILES    = %w( Rakefile ChangeLog INSTALL README LICENSE ).
-    collect {|filename| BASEDIR + filename }
-
-BUILD_FILES   = %w( convertdb.rb utils.rb )
+
+PKG_NAME      = 'wordnet'
+PKG_SUMMARY   = ''
+VERSION_FILE  = LIBDIR + 'wordnet.rb'
+PKG_VERSION   = VERSION_FILE.read[ /VERSION = '(\d+\.\d+\.\d+)'/, 1 ]
+PKG_FILE_NAME = "#{PKG_NAME.downcase}-#{PKG_VERSION}"
+GEM_FILE_NAME = "#{PKG_FILE_NAME}.gem"
+
+RELEASE_NAME  = "RELEASE_#{PKG_VERSION.gsub(/\./, '_')}"
+
+ARTIFACTS_DIR = Pathname.new( ENV['CC_BUILD_ARTIFACTS'] || 'artifacts' )
+
+TEXT_FILES    = %w( Rakefile ChangeLog README LICENSE ).collect {|filename| BASEDIR + filename }
+LIB_FILES     = Pathname.glob( LIBDIR + '**/*.rb' ).delete_if {|item| item =~ /\.svn/ }
+EXT_FILES     = Pathname.glob( EXTDIR + '**/*.{c,h,rb}' ).delete_if {|item| item =~ /\.svn/ }
 
 SPECDIR       = BASEDIR + 'spec'
-SPEC_FILES    = Pathname.glob( (SPECDIR + '**/*_spec.rb').to_s ).
-    delete_if {|item| item =~ /\.svn/ }
-# Ideally, this should be automatically generated.
+SPEC_FILES    = Pathname.glob( SPECDIR + '**/*_spec.rb' ).delete_if {|item| item =~ /\.svn/ }
 SPEC_EXCLUDES = 'spec,/Library/Ruby,/var/lib,/usr/local/lib'
 
-LIB_FILES     = Pathname.glob( LIBDIR + '**/*.rb').
-    delete_if {|item| item =~ /\.svn/ }
-
-RELEASE_FILES = TEXT_FILES + LIB_FILES + SPEC_FILES
-
+
+RELEASE_FILES = FileList[ TEXT_FILES + SPEC_FILES + LIB_FILES + EXT_FILES ]
+
+
+RCOV_OPTS = [
+    '--exclude', SPEC_EXCLUDES,
+    '--xrefs',
+    '--save',
+    '--callsites'
+  ]
+
+
+### Load some task libraries that need to be loaded early
 require RAKE_TASKDIR + 'helpers.rb'
-
-### Package constants
-PKG_NAME      = 'wordnet'
-PKG_VERSION   = find_pattern_in_file( /VERSION = '(\d+\.\d+\.\d+)'/, LIBDIR + 'wordnet.rb' ).first
-PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
-
-RELEASE_NAME  = "REL #{PKG_VERSION}"
-
-### Load task libraries
 require RAKE_TASKDIR + 'svn.rb'
 require RAKE_TASKDIR + 'verifytask.rb'
+
+# Define some constants that depend on the 'svn' tasklib
+PKG_BUILD = get_svn_rev( BASEDIR ) || 0
+SNAPSHOT_PKG_NAME = "#{PKG_FILE_NAME}.#{PKG_BUILD}"
+SNAPSHOT_GEM_NAME = "#{SNAPSHOT_PKG_NAME}.gem"
+
+# Documentation constants
+RDOC_OPTIONS = [
+    '-w', '4',
+    '-SHN',
+    '-i', '.',
+    '-m', 'README',
+    '-W', 'http://deveiate.org/projects/Ruby-WordNet/browser/trunk/'
+  ]
+
+# Release constants
+SMTP_HOST = 'mail.faeriemud.org'
+SMTP_PORT = 465 # SMTP + SSL
+
+# Project constants
+PROJECT_HOST = 'deveiate.org'
+PROJECT_PUBDIR = "/usr/local/www/public/code"
+PROJECT_DOCDIR = "#{PROJECT_PUBDIR}/#{PKG_NAME}"
+PROJECT_SCPURL = "#{PROJECT_HOST}:#{PROJECT_DOCDIR}"
+
+# Gem dependencies: gemname => version
+DEPENDENCIES = {
+#   'mongrel'       => '',
+}
+
+# Non-gem requirements: packagename => version
+REQUIREMENTS = {
+#   'Apache'  => '>= 2.2.6',
+}
+
+# RubyGem specification
+GEMSPEC   = Gem::Specification.new do |gem|
+    gem.name              = PKG_NAME.downcase
+    gem.version           = PKG_VERSION
+
+    gem.summary           = PKG_SUMMARY
+    gem.description       = <<-EOD
+
+    A Ruby implementation of the WordNet lexical dictionary
+
+    EOD
+
+    gem.authors           = 'Michael Granger'
+    gem.email             = 'ged@FaerieMUD.org'
+    gem.homepage          = 'http://deveiate.org/projects/Ruby-WordNet'
+    gem.rubyforge_project = 'deveiate'
+
+    gem.has_rdoc          = true
+    gem.rdoc_options      = RDOC_OPTIONS
+
+    gem.files             = RELEASE_FILES.
+        collect {|f| f.relative_path_from(BASEDIR).to_s }
+    gem.test_files        = SPEC_FILES.
+        collect {|f| f.relative_path_from(BASEDIR).to_s }
+        
+    DEPENDENCIES.each do |name, version|
+        version = '>= 0' if version.length.zero?
+        gem.add_dependency( name, version )
+    end
+    
+    REQUIREMENTS.each do |name, version|
+        gem.requirements << [ name, version ].compact.join(' ')
+    end
+end
+
+
+# Load any remaining task libraries
 Pathname.glob( RAKE_TASKDIR + '*.rb' ).each do |tasklib|
+    RELEASE_FILES.include( tasklib )
+
     next if tasklib =~ %r{/(helpers|svn|verifytask)\.rb$}
     begin
@@ -81 +165 @@
 end
 
-
-if Rake.application.options.trace
-    $trace = true
-    log "$trace is enabled"
-end
-
-if Rake.application.options.dryrun
-    $dryrun = true
-    log "$dryrun is enabled"
-    Rake.application.options.dryrun = false
-end
+$trace = Rake.application.options.trace ? true : false
+$dryrun = Rake.application.options.dryrun ? true : false
+
+# Load any project-specific rules defined in 'Rakefile.local' if it exists
+LOCAL_RAKEFILE = BASEDIR + 'Rakefile.local'
+if LOCAL_RAKEFILE.exist?
+    import LOCAL_RAKEFILE 
+    RELEASE_FILES.include( LOCAL_RAKEFILE.to_s )
+end
+
+
+#####################################################################
+### T A S K S   
+#####################################################################
 
 ### Default task
-task :default  => [:clean, :spec, :verify, :package]
+task :default  => [:clean, :spec, :rdoc, :package]
 
 
 ### Task: clean
-desc "Clean pkg, coverage, and rdoc; remove .bak files"
-task :clean => [ :clobber_rdoc, :clobber_package, :clobber_coverage, :clobber_manual ] do
-    files = FileList['**/*.bak']
-    files.clear_exclude
-    File.rm( files ) unless files.empty?
-    FileUtils.rm_rf( 'artifacts' )
-end
-
-
-### Task: docs -- Convenience task for rebuilding dynamic docs, including coverage, api 
-### docs, and manual
-task :docs => [ :manual, :coverage, :rdoc ]
-
-
-### Task: rdoc
-Rake::RDocTask.new do |rdoc|
-    rdoc.rdoc_dir = 'docs'
-    rdoc.title    = "Ruby WordNet"
-
-    rdoc.options += [
-        '-w', '4',
-        '-SHN',
-        '-i', 'docs',
-        '-f', 'darkfish',
-        '-m', 'README',
-        '-W', 'http://deveiate.org/projects/Ruby-WordNet/browser/trunk/'
-      ]
-    
-    rdoc.rdoc_files.include 'README'
-    rdoc.rdoc_files.include LIB_FILES.collect {|f| f.relative_path_from(BASEDIR).to_s }
-end
-
-
-### Task: gem
-gemspec = Gem::Specification.new do |gem|
-    pkg_build = get_svn_rev( BASEDIR ) || 0
-    
-    gem.name        = PKG_NAME
-    gem.version     = "%s.%s" % [ PKG_VERSION, pkg_build ]
-
-    gem.summary     = "Ruby-WordNet is a Ruby interface to the WordNet® Lexical Database"
-    gem.description = <<-EOD
-    Ruby-WordNet is a Ruby interface to the WordNet® Lexical Database. WordNet 
-    is an online lexical reference system whose design is inspired by current 
-    psycholinguistic theories of human lexical memory. English nouns, verbs, 
-    adjectives and adverbs are organized into synonym sets, each representing 
-    one underlying lexical concept. Different relations link the synonym sets.
-    EOD
-
-    gem.authors     = "Michael Granger"
-    gem.homepage    = "http://deveiate.org/projects/Ruby-WordNet/"
-
-    gem.has_rdoc    = true
-
-    gem.files       = RELEASE_FILES.
-        collect {|f| f.relative_path_from(BASEDIR).to_s }
-    gem.test_files  = SPEC_FILES.
-        collect {|f| f.relative_path_from(BASEDIR).to_s }
-
-    gem.add_dependency( 'sequel' )
-end
-Rake::GemPackageTask.new( gemspec ) do |task|
-    task.gem_spec = gemspec
-    task.need_tar = false
-    task.need_tar_gz = true
-    task.need_tar_bz2 = true
-    task.need_zip = true
-end
-
-
-### Task: install
-task :install_gem => [:package] do
-    $stderr.puts 
-    installer = Gem::Installer.new( %{pkg/#{PKG_FILE_NAME}.gem} )
-    installer.install
-end
-
-### Task: uninstall
-task :uninstall_gem => [:clean] do
-    uninstaller = Gem::Uninstaller.new( PKG_FILE_NAME )
-    uninstaller.uninstall
-end
-
+CLEAN.include 'coverage'
+CLOBBER.include 'artifacts', 'coverage.info', PKGDIR
 
 
 ### Cruisecontrol task
 desc "Cruisecontrol build"
-task :cruise => [:clean, :coverage, :package] do |task|
+task :cruise => [:clean, 'coverage:verify', :package] do |task|
     raise "Artifacts dir not set." if ARTIFACTS_DIR.to_s.empty?
     artifact_dir = ARTIFACTS_DIR.cleanpath
@@ -198 +204 @@
 
 
-### RSpec tasks
-begin
-    gem 'rspec', '>= 1.1.3'
-    require 'spec/rake/spectask'
-
-    COMMON_SPEC_OPTS = ['-c', '-f', 's']
-
-    ### Task: spec
-    Spec::Rake::SpecTask.new( :spec ) do |task|
-        task.spec_files = SPEC_FILES
-        task.libs += [LIBDIR]
-        task.spec_opts = COMMON_SPEC_OPTS
-    end
-    task :test => [:spec]
-
-
-    namespace :spec do
-        desc "Run rspec every time there's a change to one of the files"
-        task :autotest do
-            require 'autotest/rspec'
-
-            autotester = Autotest::Rspec.new
-            autotester.exceptions = %r{\.svn|\.skel}
-            autotester.run
-        end
-
-    
-        desc "Generate HTML output for a spec run"
-        Spec::Rake::SpecTask.new( :html ) do |task|
-            task.spec_files = SPEC_FILES
-            task.spec_opts = ['-f','h', '-D']
-        end
-
-        desc "Generate plain-text output for a CruiseControl.rb build"
-        Spec::Rake::SpecTask.new( :text ) do |task|
-            task.spec_files = SPEC_FILES
-            task.spec_opts = ['-f','p']
-        end
-    end
-rescue LoadError => err
-    task :no_rspec do
-        $stderr.puts "Testing tasks not defined: RSpec rake tasklib not available: %s" %
-            [ err.message ]
-    end
-    
-    task :spec => :no_rspec
-    namespace :spec do
-        task :autotest => :no_rspec
-        task :html => :no_rspec
-        task :text => :no_rspec
-    end
-end
-
-
-### RCov (via RSpec) tasks
-begin
-    gem 'rcov'
-    gem 'rspec', '>= 1.1.3'
-
-    RCOV_OPTS = ['--exclude', SPEC_EXCLUDES, '--xrefs', '--save']
-
-    ### Task: coverage (via RCov)
-    ### Task: spec
-    desc "Build test coverage reports"
-    Spec::Rake::SpecTask.new( :coverage ) do |task|
-        task.spec_files = SPEC_FILES
-        task.libs += [LIBDIR]
-        task.spec_opts = ['-f', 'p', '-b']
-        task.rcov_opts = RCOV_OPTS
-        task.rcov = true
-    end
-    
-    task :rcov => [:coverage] do; end
-    
-    ### Other coverage tasks
-    namespace :coverage do
-        desc "Generate a detailed text coverage report"
-        Spec::Rake::SpecTask.new( :text ) do |task|
-            task.spec_files = SPEC_FILES
-            task.rcov_opts = RCOV_OPTS + ['--text-report']
-            task.rcov = true
-        end
-
-        desc "Show differences in coverage from last run"
-        Spec::Rake::SpecTask.new( :diff ) do |task|
-            task.spec_files = SPEC_FILES
-            task.rcov_opts = ['--text-coverage-diff']
-            task.rcov = true
-        end
-
-        ### Task: verify coverage
-        desc "Build coverage statistics"
-        VerifyTask.new( :verify => :rcov ) do |task|
-            task.threshold = 85.0
-        end
-        
-        desc "Run RCov in 'spec-only' mode to check coverage from specs"
-        Spec::Rake::SpecTask.new( :speconly ) do |task|
-            task.spec_files = SPEC_FILES
-            task.rcov_opts = ['--exclude', SPEC_EXCLUDES, '--text-report', '--save']
-            task.rcov = true
-        end
-    end
-
-rescue LoadError => err
-    task :no_rcov do
-        $stderr.puts "Coverage tasks not defined: RSpec+RCov tasklib not available: %s" %
-            [ err.message ]
-    end
-
-    task :coverage => :no_rcov
-    task :clobber_coverage
-    task :rcov => :no_rcov
-    namespace :coverage do
-        task :text => :no_rcov
-        task :diff => :no_rcov
-    end
-    task :verify => :no_rcov
-end
-
-
-
-### Coding style checks and fixes
-namespace :style do
-    
-    BLANK_LINE = /^\s*$/
-    GOOD_INDENT = /^(\t\s*)?\S/
-
-    # A list of the files that have legitimate leading whitespace, etc.
-    PROBLEM_FILES = []
-    
-    desc "Check source files for inconsistent indent and fix them"
-    task :fix_indent do
-        files = LIB_FILES + SPEC_FILES
-
-        badfiles = Hash.new {|h,k| h[k] = [] }
-        
-        trace "Checking files for indentation"
-        files.each do |file|
-            if PROBLEM_FILES.include?( file )
-                trace "  skipping problem file #{file}..."
-                next
-            end
-            
-            trace "  #{file}"
-            linecount = 0
-            file.each_line do |line|
-                linecount += 1
-                
-                # Skip blank lines
-                next if line =~ BLANK_LINE
-                
-                # If there's a line with incorrect indent, note it and skip to the 
-                # next file
-                if line !~ GOOD_INDENT
-                    trace "    Bad line %d: %p" % [ linecount, line ]
-                    badfiles[file] << [ linecount, line ]
-                end
-            end
-        end
-
-        if badfiles.empty?
-            log "No indentation problems found."
-        else
-            log "Found incorrect indent in #{badfiles.length} files:\n  "
-            badfiles.each do |file, badlines|
-                log "  #{file}:\n" +
-                    "    " + badlines.collect {|badline| "%5d: %p" % badline }.join( "\n    " )
-            end
-        end
-    end
-
-end
-
-

branches/rakefile-work/convertdb.rb

@@ -50 +50 @@
 $scanner = StringScanner::new( "" )
 
-# Source WordNet files
-IndexFiles = %w[ index.noun index.verb index.adj index.adv ]
-MorphFiles = {
-    'adj.exc'       => WordNet::Adjective,
-    'adv.exc'       => WordNet::Adverb,
-    'noun.exc'      => WordNet::Noun,
-    'verb.exc'      => WordNet::Verb,
-    'cousin.exc'    => '',
-}
-DataFiles =  {
-    'data.adj'      => WordNet::Adjective,
-    'data.adv'      => WordNet::Adverb,
-    'data.noun'     => WordNet::Noun,
-    'data.verb'     => WordNet::Verb,
-}
-
-# Struct which represents a list of files, a database, and a processor function
-# for moving records from each of the files into the database.
-Fileset = Struct::new( "WordNetFileset", :files, :name, :db, :processor )
-
-# How many records to insert between commits
-CommitThreshold = 2000
-
-# Temporary location for the lexicon data files
-BuildDir = Pathname.new( __FILE__ ).expand_path.dirname + 
-           Pathname.new( WordNet::Lexicon::DEFAULT_DB_ENV ).basename
-
-
-
-#####################################################################
-### M A I N   P R O G R A M
-#####################################################################
-def convertdb( errorLimit=0 )
-    $stderr.sync = $stdout.sync = true
-    header "WordNet Lexicon Converter"
-
-    # Make sure the user knows what they're in for
-    message "This program will convert WordNet data files into databases\n"\
-        "used by Ruby-WordNet. This will not affect existing WordNet files,\n"\
-        "but will require up to 40Mb of disk space.\n"
-    exit unless /^y/i =~ prompt_with_default("Continue?", "y")
-
-    # Open the database and check to be sure it's empty. Confirm overwrite if
-    # not. Checkpoint and set up logging proc if debugging.
-    if BuildDir.exist?
-        message ">>> Warning: Existing data in the Ruby-WordNet databases\n"\
-            "will be overwritten.\n"
-        abort( "user cancelled." ) unless 
-            /^y/i =~ prompt_with_default( "Continue?", "n" )
-        BuildDir.rmtree
+class WordNetConverter
+
+    # Source WordNet files
+    IndexFiles = %w[ index.noun index.verb index.adj index.adv ]
+    MorphFiles = {
+        'adj.exc'       => WordNet::Adjective,
+        'adv.exc'       => WordNet::Adverb,
+        'noun.exc'      => WordNet::Noun,
+        'verb.exc'      => WordNet::Verb,
+        'cousin.exc'    => '',
+    }
+    DataFiles =  {
+        'data.adj'      => WordNet::Adjective,
+        'data.adv'      => WordNet::Adverb,
+        'data.noun'     => WordNet::Noun,
+        'data.verb'     => WordNet::Verb,
+    }
+
+    # Struct which represents a list of files, a database, and a processor function
+    # for moving records from each of the files into the database.
+    Fileset = Struct::new( "WordNetFileset", :files, :name, :db, :processor )
+
+    # How many records to insert between commits
+    CommitThreshold = 2000
+
+    # Temporary location for the lexicon data files
+    BuildDir = Pathname.new( __FILE__ ).expand_path.dirname + 
+               Pathname.new( WordNet::Lexicon::DEFAULT_DB_ENV ).basename
+
+
+    ### Create a new converter that will dump WordNet dictionary files into a BerkeleyDB 
+    ### in the given +builddir+ 
+    def initialize( builddir=BuildDir )
+        @builddir = Pathname.new( builddir )
     end
-
-    # Find the source data files
-    if ARGV.empty?
+    
+
+    ### Convert the various dict files from the WordNet project into a BerkeleyDB database
+    def convertdb( errorLimit=0 )
+        $stderr.sync = $stdout.sync = true
+        header "WordNet Lexicon Converter"
+
+        # Make sure the user knows what they're in for
+        message "This program will convert WordNet data files into databases\n"\
+            "used by Ruby-WordNet. This will not affect existing WordNet files,\n"\
+            "but will require up to 40Mb of disk space.\n"
+        exit unless /^y/i =~ prompt_with_default("Continue?", "y")
+
+        # Open the database and check to be sure it's empty. Confirm overwrite if
+        # not. Checkpoint and set up logging proc if debugging.
+        if @builddir.exist? && @builddir.entries.include?( 'data' )
+            message ">>> Warning: Existing data in the Ruby-WordNet databases\n"\
+                "will be overwritten.\n"
+            abort( "user cancelled." ) unless 
+                /^y/i =~ prompt_with_default( "Continue?", "n" )
+            @builddir.rmtree
+        end
+
+        # Find the source data files
         default = nil
-        
-        if wndirs = Pathname.glob( Pathname.getwd + 'WordNet-*' )
+        wndirs = Pathname.glob( Pathname.getwd + 'WordNet-*' )
+        unless wndirs.empty?
             default = wndirs.first
         else
@@ -114 +119 @@
         # :TODO: Do some more intelligent searching here
         message "Where can I find the WordNet data files?\n"
-        datadir = prompt_with_default( "Data directory", default + "dict" )
-    else
-        datadir = ARGV.shift
-    end
-    datadir = Pathname.new( datadir )
-
-    abort( "Directory '#{datadir}' does not exist" ) unless datadir.exist?
-    abort( "'#{datadir}' is not a directory" ) unless datadir.directory?
-    testfile = datadir + "data.noun"
-    abort( "'#{datadir}' doesn't seem to contain the necessary files.") unless testfile.exist?
-
-    # Open the lexicon readwrite into the temporary datadir
-    BuildDir.mkpath
-    lexicon = WordNet::Lexicon::new( BuildDir.to_s, 0666 )
-
-    # Process each fileset
-    [     # Fileset,  name,    database handle, processor
-        Fileset::new( IndexFiles, "index", lexicon.index_db, method(:parse_index_line) ),
-        Fileset::new( MorphFiles, "morph", lexicon.morph_db, method(:parse_morph_line) ),
-        Fileset::new( DataFiles,  "data",  lexicon.data_db,  method(:parse_synset_line) ),
-    ].each do |set|
-        message "Converting %s files...\n" % set.name
-        set.db.truncate
-
-        # Process each file in the set with the appropriate processor method and
-        # insert results into the corresponding table.
-        set.files.each do |file,pos|
-            message "    #{file}..."
-
-            filepath = File::join( datadir, file )
-            if !File::exists?( filepath )
-                message "missing: skipped\n"
-                next
-            end
-
-            txn, dbh = lexicon.env.txn_begin( 0, set.db )
-            entries = lineNumber = errors = 0
-            File::readlines( filepath ).each do |line|
-                lineNumber += 1
-                next if /^\s/ =~ line
-
-                key, value = set.processor.call( line.chomp, lineNumber, pos )
-                unless key
-                    errors += 1
-                    if errorLimit.nonzero? && errors >= errorLimit
-                        abort( "Too many errors" )
+        datadir = prompt_with_default( "Data directory", default + "/dict" )
+        datadir = Pathname.new( datadir )
+
+        abort( "Directory '#{datadir}' does not exist" ) unless datadir.exist?
+        abort( "'#{datadir}' is not a directory" ) unless datadir.directory?
+        testfile = datadir + "data.noun"
+        abort( "'#{datadir}' doesn't seem to contain the necessary files.") unless testfile.exist?
+
+        # Open the lexicon readwrite into the temporary datadir
+        @builddir.mkpath
+        lexicon = WordNet::Lexicon::new( @builddir.to_s, 0666 )
+
+        # Process each fileset
+        [     # Fileset,  name,    database handle, processor
+            Fileset::new( IndexFiles, "index", lexicon.index_db, method(:parse_index_line) ),
+            Fileset::new( MorphFiles, "morph", lexicon.morph_db, method(:parse_morph_line) ),
+            Fileset::new( DataFiles,  "data",  lexicon.data_db,  method(:parse_synset_line) ),
+        ].each do |set|
+            message "Converting %s files...\n" % set.name
+            set.db.truncate
+
+            # Process each file in the set with the appropriate processor method and
+            # insert results into the corresponding table.
+            set.files.each do |file,pos|
+                message "    #{file}..."
+
+                filepath = File::join( datadir, file )
+                if !File::exists?( filepath )
+                    message "missing: skipped\n"
+                    next
+                end
+
+                txn, dbh = lexicon.env.txn_begin( 0, set.db )
+                entries = lineNumber = errors = 0
+                File::readlines( filepath ).each do |line|
+                    lineNumber += 1
+                    next if /^\s/ =~ line
+
+                    key, value = set.processor.call( line.chomp, lineNumber, pos )
+                    unless key
+                        errors += 1
+                        if errorLimit.nonzero? && errors >= errorLimit
+                            abort( "Too many errors" )
+                        end
+                    end
+
+                    dbh[ key ] = value
+                    entries += 1
+                    print "%d%s" % [ entries, "\x08" * entries.to_s.length ]
+
+                    # Commit and start a new transaction every 1000 records
+                    if (entries % CommitThreshold).zero?

@@ -1 @@
-rake svn://deveiate.org/rake-tasklibs/trunks
+rake svn://deveiate.org/rake-tasklibs/trunk