Board index » jbuilder » memory problems with DB import

memory problems with DB import


2005-07-20 01:43:55 PM
jbuilder12
Dear all
I have written an small tool that takes a tab-delimited text files and
imports them into a PostgreSQL table. The import as such would run well,
but I get a severe memory problem. The tool should import around 250
text files, with the largest being 70 Megs.
My problem is that the application does not seem to manage memory very
well or I have introduced a memory leak somewhere. This snipped of code
causes the problem:
<---Code Snippet------------------------------------------------------>
outputArea.append("Importing...:");
database1.openConnection();
for (int i = 0; i < fileNames.length; i++)
{
String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
textDataFile.setFileName(absolutePath);
textDataFile.load(tableDataSet1);
System.out.println("Current file: " + absolutePath);
outputArea.append("\nRows loaded: " + tableDataSet1.getRowCount());
database1.saveChanges(tableDataSet1);
tableDataSet1.clearValues();
}
database1.closeConnection();
outputArea.append("\nImport completed!");
<---Code Snippet------------------------------------------------------>
Does anyone know what I am doing wrong here? How can I improve memory
consumption?
Here is the full class:
<---Code Snippet------------------------------------------------------>
package com.shiftthink.geotools.data.geonames;
import java.io.*;
import com.borland.dx.dataset.*;
import com.borland.dx.sql.dataset.*;
import javax.swing.JTextArea;
import javax.swing.JProgressBar;
import java.util.Arrays;
public class DBHelper
{
static String[] fileNames =
{
"ac", "al", "ag", "an", "ao", "av", "af", "ar", "am", "aa", "at",
"as",
"au", "aj", "bf", "ba", "bg", "bb", "bs", "bo", "be", "bh", "bn",
"bd",
"bt", "bl", "bk", "bc", "bv", "br", "io", "vi", "bx", "bu", "uv",
"bm",
"by", "cb", "cm", "ca", "cv", "cj", "ct", "cd", "ci", "ch", "kt",
"ip",
"ck", "co", "cn", "cf", "cg", "cw", "cr", "cs", "iv", "hr", "cu",
"cy",
"ez", "da", "dj", "do", "dr", "tt", "ec", "eg", "es", "ek", "er",
"en",
"et", "eu", "fk", "fo", "fj", "fi", "fr", "fg", "fp", "fs", "gb",
"ga",
"gz", "gg", "gm", "gh", "gi", "go", "gr", "gl", "gj", "gp", "gt",
"gt",
"gk", "pu", "gy", "ha", "hm", "ho", "hk", "hu", "ic", "in", "id",
"ir",
"iz", "ei", "im", "is", "it", "jm", "jn", "ja", "je", "jo", "ju",
"kz",
"ke", "kr", "ku", "kg", "la", "lg", "le", "lt", "li", "ly", "ls",
"lh",
"lu", "mc", "mk", "ma", "mi", "my", "mv", "ml", "mt", "rm", "mb",
"mr",
"mp", "mf", "mx", "fm", "md", "mn", "mg", "mh", "mo", "mz", "wa",
"nr",
"np", "nl", "nt", "nc", "nz", "nu", "ng", "ni", "ne", "nm", "nf",
"kn",
"no", "os", "mu", "pk", "ps", "pm", "pp", "pf", "pa", "pe", "rp",
"pc",
"pl", "po", "qa", "re", "ro", "rs", "rw", "sh", "sc", "st", "sb",
"vc",
"ws", "sm", "tp", "sa", "sg", "yi", "se", "sl", "sn", "lo", "si",
"bp",
"so", "sf", "sx", "ks", "sp", "pg", "ce", "su", "ns", "sv", "wz",
"sw",
"sz", "sy", "tw", "ti", "tz", "th", "to", "tl", "tn", "td", "te",
"ts",
"tu", "tx", "tk", "tv", "ug", "up", "uf", "ae", "uk", "uy", "uz",
"nh",
"vt", "ve", "vm", "wf", "we", "wi", "ym", "za", "zi"
};
TextDataFile textDataFile = new TextDataFile();
TableDataSet tableDataSet1 = new TableDataSet();
Column RC = new Column();
Column UFI = new Column();
Column UNI = new Column();
Column LAT = new Column();
Column LONG = new Column();
Column DMS_LAT = new Column();
Column DMS_LONG = new Column();
Column UTM = new Column();
Column JOG = new Column();
Column FC = new Column();
Column DSG = new Column();
Column PC = new Column();
Column CC1 = new Column();
Column ADM1 = new Column();
Column ADM2 = new Column();
Column DIM = new Column();
Column CC2 = new Column();
Column NT = new Column();
Column LC = new Column();
Column SHORT_FORM = new Column();
Column GENERIC = new Column();
Column SHORT_NAME = new Column();
Column FULL_NAME = new Column();
Column FULL_NAME_ND = new Column();
Column MOD_DATE = new Column();
Database database1 = new Database();
public DBHelper()
{
}
public void gnImport(String filePath, JTextArea outputArea) throws
Exception
{
database1.setConnection(new
com.borland.dx.sql.dataset.ConnectionDescriptor("jdbc:postgresql:ST_EIS",
"st_eis_user", "fairline", false, "org.postgresql.Driver"));
MOD_DATE.setCaption("MOD_DATE");
MOD_DATE.setColumnName("MOD_DATE");
MOD_DATE.setDataType(com.borland.dx.dataset.Variant.STRING);
MOD_DATE.setPreferredOrdinal(24);
MOD_DATE.setWidth(15);
MOD_DATE.setServerColumnName("MOD_DATE");
MOD_DATE.setSqlType(0);
FULL_NAME_ND.setCaption("FULL_NAME_ND");
FULL_NAME_ND.setColumnName("FULL_NAME_ND");
FULL_NAME_ND.setDataType(com.borland.dx.dataset.Variant.STRING);
FULL_NAME_ND.setPreferredOrdinal(23);
FULL_NAME_ND.setWidth(200);
FULL_NAME_ND.setServerColumnName("FULL_NAME_ND");
FULL_NAME_ND.setSqlType(0);
FULL_NAME.setCaption("FULL_NAME");
FULL_NAME.setColumnName("FULL_NAME");
FULL_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
FULL_NAME.setPreferredOrdinal(22);
FULL_NAME.setWidth(200);
FULL_NAME.setServerColumnName("FULL_NAME");
FULL_NAME.setSqlType(0);
SHORT_NAME.setCaption("SHORT_NAME");
SHORT_NAME.setColumnName("SHORT_NAME");
SHORT_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
SHORT_NAME.setPreferredOrdinal(21);
SHORT_NAME.setWidth(200);
SHORT_NAME.setServerColumnName("SHORT_NAME");
SHORT_NAME.setSqlType(0);
GENERIC.setCaption("GENERIC");
GENERIC.setColumnName("GENERIC");
GENERIC.setDataType(com.borland.dx.dataset.Variant.STRING);
GENERIC.setPreferredOrdinal(20);
GENERIC.setWidth(128);
GENERIC.setServerColumnName("GENERIC");
GENERIC.setSqlType(0);
SHORT_FORM.setCaption("SHORT_FORM");
SHORT_FORM.setColumnName("SHORT_FORM");
SHORT_FORM.setDataType(com.borland.dx.dataset.Variant.STRING);
SHORT_FORM.setPreferredOrdinal(19);
SHORT_FORM.setWidth(128);
SHORT_FORM.setServerColumnName("SHORT_FORM");
SHORT_FORM.setSqlType(0);
LC.setCaption("LC");
LC.setColumnName("LC");
LC.setDataType(com.borland.dx.dataset.Variant.STRING);
LC.setPreferredOrdinal(18);
LC.setWidth(2);
LC.setServerColumnName("LC");
LC.setSqlType(0);
NT.setCaption("NT");
NT.setColumnName("NT");
NT.setDataType(com.borland.dx.dataset.Variant.STRING);
NT.setPreferredOrdinal(17);
NT.setWidth(1);
NT.setServerColumnName("NT");
NT.setSqlType(0);
CC2.setCaption("CC2");
CC2.setColumnName("CC2");
CC2.setDataType(com.borland.dx.dataset.Variant.STRING);
CC2.setPreferredOrdinal(16);
CC2.setWidth(3);
CC2.setServerColumnName("CC2");
CC2.setSqlType(0);
DIM.setCaption("DIM");
DIM.setColumnName("DIM");
DIM.setDataType(com.borland.dx.dataset.Variant.STRING);
DIM.setPreferredOrdinal(15);
DIM.setWidth(15);
DIM.setServerColumnName("DIM");
DIM.setSqlType(0);
ADM2.setCaption("ADM2");
ADM2.setColumnName("ADM2");
ADM2.setDataType(com.borland.dx.dataset.Variant.STRING);
ADM2.setPreferredOrdinal(14);
ADM2.setWidth(200);
ADM2.setServerColumnName("ADM2");
ADM2.setSqlType(0);
ADM1.setCaption("ADM1");
ADM1.setColumnName("ADM1");
ADM1.setDataType(com.borland.dx.dataset.Variant.STRING);
ADM1.setPreferredOrdinal(13);
ADM1.setWidth(2);
ADM1.setServerColumnName("ADM1");
ADM1.setSqlType(0);
CC1.setCaption("CC1");
CC1.setColumnName("CC1");
CC1.setDataType(com.borland.dx.dataset.Variant.STRING);
CC1.setPreferredOrdinal(12);
CC1.setWidth(3);
CC1.setServerColumnName("CC1");
CC1.setSqlType(0);
PC.setCaption("PC");
PC.setColumnName("PC");
PC.setDataType(com.borland.dx.dataset.Variant.STRING);
PC.setPreferredOrdinal(11);
PC.setWidth(2);
PC.setServerColumnName("PC");
PC.setSqlType(0);
DSG.setCaption("DSG");
DSG.setColumnName("DSG");
DSG.setDataType(com.borland.dx.dataset.Variant.STRING);
DSG.setPreferredOrdinal(10);
DSG.setWidth(12);
DSG.setServerColumnName("DSG");
DSG.setSqlType(0);
FC.setCaption("FC");
FC.setColumnName("FC");
FC.setDataType(com.borland.dx.dataset.Variant.STRING);
FC.setPreferredOrdinal(9);
FC.setWidth(2);
FC.setServerColumnName("FC");
FC.setSqlType(0);
JOG.setCaption("JOG");
JOG.setColumnName("JOG");
JOG.setDataType(com.borland.dx.dataset.Variant.STRING);
JOG.setPreferredOrdinal(8);
JOG.setWidth(10);
JOG.setServerColumnName("JOG");
JOG.setSqlType(0);
UTM.setCaption("UTM");
UTM.setColumnName("UTM");
UTM.setDataType(com.borland.dx.dataset.Variant.STRING);
UTM.setPreferredOrdinal(7);
UTM.setWidth(5);
UTM.setServerColumnName("UTM");
UTM.setSqlType(0);
DMS_LONG.setCaption("DMS_LONG");
DMS_LONG.setColumnName("DMS_LONG");
DMS_LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
DMS_LONG.setPreferredOrdinal(6);
DMS_LONG.setServerColumnName("DMS_LONG");
DMS_LONG.setWidth(15);
DMS_LONG.setSqlType(0);
DMS_LAT.setCaption("DMS_LAT");
DMS_LAT.setColumnName("DMS_LAT");
DMS_LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
DMS_LAT.setPreferredOrdinal(5);
DMS_LAT.setServerColumnName("DMS_LAT");
DMS_LAT.setWidth(15);
DMS_LAT.setSqlType(0);
LONG.setCaption("LONG");
LONG.setColumnName("LONG");
LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
LONG.setPreferredOrdinal(4);
LONG.setServerColumnName("LONG");
LONG.setWidth(15);
LONG.setSqlType(0);
LAT.setCaption("LAT");
LAT.setColumnName("LAT");
LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
LAT.setPreferredOrdinal(3);
LAT.setServerColumnName("LAT");
LAT.setWidth(15);
LAT.setSqlType(0);
UNI.setCaption("UNI");
UNI.setColumnName("UNI");
UNI.setDataType(com.borland.dx.dataset.Variant.STRING);
UNI.setPreferredOrdinal(2);
UNI.setServerColumnName("UNI");
UNI.setWidth(15);
UNI.setSqlType(0);
UFI.setCaption("UFI");
UFI.setColumnName("UFI");
UFI.setDataType(com.borland.dx.dataset.Variant.STRING);
UFI.setPreferredOrdinal(1);
UFI.setServerColumnName("UFI");
UFI.setWidth(15);
UFI.setSqlType(0);
RC.setCaption("RC");
RC.setColumnName("RC");
RC.setDataType(com.borland.dx.dataset.Variant.STRING);
RC.setPreferredOrdinal(0);
RC.setWidth(1);
RC.setServerColumnName("RC");
RC.setSqlType(0);
tableDataSet1.setColumns(new Column[]
{RC, UFI, UNI, LAT, LONG, DMS_LAT,
DMS_LONG, UTM,
JOG, FC, DSG, PC, CC1, ADM1, ADM2, DIM,
CC2, NT,
LC, SHORT_FORM, GENERIC, SHORT_NAME, FULL_NAME,
FULL_NAME_ND, MOD_DATE});
tableDataSet1.setTableName("coordinates");
tableDataSet1.setSchemaName("st_eis");
textDataFile.setDelimiter("");
textDataFile.setEncoding("Cp1252");
textDataFile.setFileFormat(com.borland.dx.dataset.DataFileFormat.ASCII);
textDataFile.setSeparator("\t");
textDataFile.setLoadAsInserted(true);
try
{
/**
==================================================================*/
/** Removing first line (headers) */
/**
==================================================================*/
for (int i = 0; i < fileNames.length; i++)
{
String removerPath = filePath + "\\" + fileNames[i] + ".txt";
RandomAccessFile file = new RandomAccessFile(removerPath, "rws");
file.seek(0);
String line1 = file.readLine();
byte content[] = new byte[line1.length()];
file.seek(0);
file.write(content);
file.close();
}
// ==================================================================
// Importing
// ==================================================================
outputArea.append("Importing...:");
database1.openConnection();
for (int i = 0; i < fileNames.length; i++)
{
String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
textDataFile.setFileName(absolutePath);
textDataFile.load(tableDataSet1);
System.out.println("Current file: " + absolutePath);
outputArea.append("\nRows loaded: " + tableDataSet1.getRowCount());
database1.saveChanges(tableDataSet1);
tableDataSet1.clearValues();
}
database1.closeConnection();
outputArea.append("\nImport completed!");
}
catch (Exception ex)
{
ex.printStackTrace();
throw new Exception();
}
}
}
<---Code Snippet------------------------------------------------------>
Thanks,
Michael
 
 

Re:memory problems with DB import

Michael,
Other than being a "very wide" table and therefore a LOT of column
objects.. Nothing is obviously off..
Have you tried giving the VM more memory..??
John..
Michael Boeni wrote:
Quote
Dear all

I have written an small tool that takes a tab-delimited text files and
imports them into a PostgreSQL table. The import as such would run well,
but I get a severe memory problem. The tool should import around 250
text files, with the largest being 70 Megs.

My problem is that the application does not seem to manage memory very
well or I have introduced a memory leak somewhere. This snipped of code
causes the problem:

<---Code Snippet------------------------------------------------------>
outputArea.append("Importing...:");
database1.openConnection();
for (int i = 0; i < fileNames.length; i++)
{

String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
textDataFile.setFileName(absolutePath);
textDataFile.load(tableDataSet1);
System.out.println("Current file: " + absolutePath);
outputArea.append("\nRows loaded: " + tableDataSet1.getRowCount());
database1.saveChanges(tableDataSet1);
tableDataSet1.clearValues();

}
database1.closeConnection();
outputArea.append("\nImport completed!");

<---Code Snippet------------------------------------------------------>

Does anyone know what I am doing wrong here? How can I improve memory
consumption?

Here is the full class:

<---Code Snippet------------------------------------------------------>
package com.shiftthink.geotools.data.geonames;

import java.io.*;
import com.borland.dx.dataset.*;
import com.borland.dx.sql.dataset.*;
import javax.swing.JTextArea;
import javax.swing.JProgressBar;
import java.util.Arrays;

public class DBHelper
{
static String[] fileNames =
{
"ac", "al", "ag", "an", "ao", "av", "af", "ar", "am", "aa", "at",
"as",
"au", "aj", "bf", "ba", "bg", "bb", "bs", "bo", "be", "bh", "bn",
"bd",
"bt", "bl", "bk", "bc", "bv", "br", "io", "vi", "bx", "bu", "uv",
"bm",
"by", "cb", "cm", "ca", "cv", "cj", "ct", "cd", "ci", "ch", "kt",
"ip",
"ck", "co", "cn", "cf", "cg", "cw", "cr", "cs", "iv", "hr", "cu",
"cy",
"ez", "da", "dj", "do", "dr", "tt", "ec", "eg", "es", "ek", "er",
"en",
"et", "eu", "fk", "fo", "fj", "fi", "fr", "fg", "fp", "fs", "gb",
"ga",
"gz", "gg", "gm", "gh", "gi", "go", "gr", "gl", "gj", "gp", "gt",
"gt",
"gk", "pu", "gy", "ha", "hm", "ho", "hk", "hu", "ic", "in", "id",
"ir",
"iz", "ei", "im", "is", "it", "jm", "jn", "ja", "je", "jo", "ju",
"kz",
"ke", "kr", "ku", "kg", "la", "lg", "le", "lt", "li", "ly", "ls",
"lh",
"lu", "mc", "mk", "ma", "mi", "my", "mv", "ml", "mt", "rm", "mb",
"mr",
"mp", "mf", "mx", "fm", "md", "mn", "mg", "mh", "mo", "mz", "wa",
"nr",
"np", "nl", "nt", "nc", "nz", "nu", "ng", "ni", "ne", "nm", "nf",
"kn",
"no", "os", "mu", "pk", "ps", "pm", "pp", "pf", "pa", "pe", "rp",
"pc",
"pl", "po", "qa", "re", "ro", "rs", "rw", "sh", "sc", "st", "sb",
"vc",
"ws", "sm", "tp", "sa", "sg", "yi", "se", "sl", "sn", "lo", "si",
"bp",
"so", "sf", "sx", "ks", "sp", "pg", "ce", "su", "ns", "sv", "wz",
"sw",
"sz", "sy", "tw", "ti", "tz", "th", "to", "tl", "tn", "td", "te",
"ts",
"tu", "tx", "tk", "tv", "ug", "up", "uf", "ae", "uk", "uy", "uz",
"nh",
"vt", "ve", "vm", "wf", "we", "wi", "ym", "za", "zi"
};
TextDataFile textDataFile = new TextDataFile();
TableDataSet tableDataSet1 = new TableDataSet();
Column RC = new Column();
Column UFI = new Column();
Column UNI = new Column();
Column LAT = new Column();
Column LONG = new Column();
Column DMS_LAT = new Column();
Column DMS_LONG = new Column();
Column UTM = new Column();
Column JOG = new Column();
Column FC = new Column();
Column DSG = new Column();
Column PC = new Column();
Column CC1 = new Column();
Column ADM1 = new Column();
Column ADM2 = new Column();
Column DIM = new Column();
Column CC2 = new Column();
Column NT = new Column();
Column LC = new Column();
Column SHORT_FORM = new Column();
Column GENERIC = new Column();
Column SHORT_NAME = new Column();
Column FULL_NAME = new Column();
Column FULL_NAME_ND = new Column();
Column MOD_DATE = new Column();
Database database1 = new Database();


public DBHelper()
{

}


public void gnImport(String filePath, JTextArea outputArea) throws
Exception
{
database1.setConnection(new
com.borland.dx.sql.dataset.ConnectionDescriptor("jdbc:postgresql:ST_EIS",
"st_eis_user", "fairline", false, "org.postgresql.Driver"));
MOD_DATE.setCaption("MOD_DATE");
MOD_DATE.setColumnName("MOD_DATE");
MOD_DATE.setDataType(com.borland.dx.dataset.Variant.STRING);
MOD_DATE.setPreferredOrdinal(24);
MOD_DATE.setWidth(15);
MOD_DATE.setServerColumnName("MOD_DATE");
MOD_DATE.setSqlType(0);
FULL_NAME_ND.setCaption("FULL_NAME_ND");
FULL_NAME_ND.setColumnName("FULL_NAME_ND");
FULL_NAME_ND.setDataType(com.borland.dx.dataset.Variant.STRING);
FULL_NAME_ND.setPreferredOrdinal(23);
FULL_NAME_ND.setWidth(200);
FULL_NAME_ND.setServerColumnName("FULL_NAME_ND");
FULL_NAME_ND.setSqlType(0);
FULL_NAME.setCaption("FULL_NAME");
FULL_NAME.setColumnName("FULL_NAME");
FULL_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
FULL_NAME.setPreferredOrdinal(22);
FULL_NAME.setWidth(200);
FULL_NAME.setServerColumnName("FULL_NAME");
FULL_NAME.setSqlType(0);
SHORT_NAME.setCaption("SHORT_NAME");
SHORT_NAME.setColumnName("SHORT_NAME");
SHORT_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
SHORT_NAME.setPreferredOrdinal(21);
SHORT_NAME.setWidth(200);
SHORT_NAME.setServerColumnName("SHORT_NAME");
SHORT_NAME.setSqlType(0);
GENERIC.setCaption("GENERIC");
GENERIC.setColumnName("GENERIC");
GENERIC.setDataType(com.borland.dx.dataset.Variant.STRING);
GENERIC.setPreferredOrdinal(20);
GENERIC.setWidth(128);
GENERIC.setServerColumnName("GENERIC");
GENERIC.setSqlType(0);
SHORT_FORM.setCaption("SHORT_FORM");
SHORT_FORM.setColumnName("SHORT_FORM");
SHORT_FORM.setDataType(com.borland.dx.dataset.Variant.STRING);
SHORT_FORM.setPreferredOrdinal(19);
SHORT_FORM.setWidth(128);
SHORT_FORM.setServerColumnName("SHORT_FORM");
SHORT_FORM.setSqlType(0);
LC.setCaption("LC");
LC.setColumnName("LC");
LC.setDataType(com.borland.dx.dataset.Variant.STRING);
LC.setPreferredOrdinal(18);
LC.setWidth(2);
LC.setServerColumnName("LC");
LC.setSqlType(0);
NT.setCaption("NT");
NT.setColumnName("NT");
NT.setDataType(com.borland.dx.dataset.Variant.STRING);
NT.setPreferredOrdinal(17);
NT.setWidth(1);
NT.setServerColumnName("NT");
NT.setSqlType(0);
CC2.setCaption("CC2");
CC2.setColumnName("CC2");
CC2.setDataType(com.borland.dx.dataset.Variant.STRING);
CC2.setPreferredOrdinal(16);
CC2.setWidth(3);
CC2.setServerColumnName("CC2");
CC2.setSqlType(0);
DIM.setCaption("DIM");
DIM.setColumnName("DIM");
DIM.setDataType(com.borland.dx.dataset.Variant.STRING);
DIM.setPreferredOrdinal(15);
DIM.setWidth(15);
DIM.setServerColumnName("DIM");
DIM.setSqlType(0);
ADM2.setCaption("ADM2");
ADM2.setColumnName("ADM2");
ADM2.setDataType(com.borland.dx.dataset.Variant.STRING);
ADM2.setPreferredOrdinal(14);
ADM2.setWidth(200);
ADM2.setServerColumnName("ADM2");
ADM2.setSqlType(0);
ADM1.setCaption("ADM1");
ADM1.setColumnName("ADM1");
ADM1.setDataType(com.borland.dx.dataset.Variant.STRING);
ADM1.setPreferredOrdinal(13);
ADM1.setWidth(2);
ADM1.setServerColumnName("ADM1");
ADM1.setSqlType(0);
CC1.setCaption("CC1");
CC1.setColumnName("CC1");
CC1.setDataType(com.borland.dx.dataset.Variant.STRING);
CC1.setPreferredOrdinal(12);
CC1.setWidth(3);
CC1.setServerColumnName("CC1");
CC1.setSqlType(0);
PC.setCaption("PC");
PC.setColumnName("PC");
PC.setDataType(com.borland.dx.dataset.Variant.STRING);
PC.setPreferredOrdinal(11);
PC.setWidth(2);
PC.setServerColumnName("PC");
PC.setSqlType(0);
DSG.setCaption("DSG");
DSG.setColumnName("DSG");
DSG.setDataType(com.borland.dx.dataset.Variant.STRING);
DSG.setPreferredOrdinal(10);
DSG.setWidth(12);
DSG.setServerColumnName("DSG");
DSG.setSqlType(0);
FC.setCaption("FC");
FC.setColumnName("FC");
FC.setDataType(com.borland.dx.dataset.Variant.STRING);
FC.setPreferredOrdinal(9);
FC.setWidth(2);
FC.setServerColumnName("FC");
FC.setSqlType(0);
JOG.setCaption("JOG");
JOG.setColumnName("JOG");
JOG.setDataType(com.borland.dx.dataset.Variant.STRING);
JOG.setPreferredOrdinal(8);
JOG.setWidth(10);
JOG.setServerColumnName("JOG");
JOG.setSqlType(0);
UTM.setCaption("UTM");
UTM.setColumnName("UTM");
UTM.setDataType(com.borland.dx.dataset.Variant.STRING);
UTM.setPreferredOrdinal(7);
UTM.setWidth(5);
UTM.setServerColumnName("UTM");
UTM.setSqlType(0);
DMS_LONG.setCaption("DMS_LONG");
DMS_LONG.setColumnName("DMS_LONG");
DMS_LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
DMS_LONG.setPreferredOrdinal(6);
DMS_LONG.setServerColumnName("DMS_LONG");
DMS_LONG.setWidth(15);
DMS_LONG.setSqlType(0);
DMS_LAT.setCaption("DMS_LAT");
DMS_LAT.setColumnName("DMS_LAT");
DMS_LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
DMS_LAT.setPreferredOrdinal(5);
DMS_LAT.setServerColumnName("DMS_LAT");
DMS_LAT.setWidth(15);
DMS_LAT.setSqlType(0);
LONG.setCaption("LONG");
LONG.setColumnName("LONG");
LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
LONG.setPreferredOrdinal(4);
LONG.setServerColumnName("LONG");
LONG.setWidth(15);
LONG.setSqlType(0);
LAT.setCaption("LAT");
LAT.setColumnName("LAT");
LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
LAT.setPreferredOrdinal(3);
LAT.setServerColumnName("LAT");
LAT.setWidth(15);
LAT.setSqlType(0);
UNI.setCaption("UNI");
UNI.setColumnName("UNI");
UNI.setDataType(com.borland.dx.dataset.Variant.STRING);
UNI.setPreferredOrdinal(2);
UNI.setServerColumnName("UNI");
UNI.setWidth(15);
UNI.setSqlType(0);
UFI.setCaption("UFI");
UFI.setColumnName("UFI");
UFI.setDataType(com.borland.dx.dataset.Variant.STRING);
UFI.setPreferredOrdinal(1);
UFI.setServerColumnName("UFI");
UFI.setWidth(15);
UFI.setSqlType(0);
RC.setCaption("RC");
RC.setColumnName("RC");
RC.setDataType(com.borland.dx.dataset.Variant.STRING);
RC.setPreferredOrdinal(0);
RC.setWidth(1);
RC.setServerColumnName("RC");
RC.setSqlType(0);


tableDataSet1.setColumns(new Column[]
{RC, UFI, UNI, LAT, LONG, DMS_LAT,
DMS_LONG, UTM,
JOG, FC, DSG, PC, CC1, ADM1, ADM2, DIM,
CC2, NT,
LC, SHORT_FORM, GENERIC, SHORT_NAME, FULL_NAME,
FULL_NAME_ND, MOD_DATE});
tableDataSet1.setTableName("coordinates");
tableDataSet1.setSchemaName("st_eis");

textDataFile.setDelimiter("");
textDataFile.setEncoding("Cp1252");
textDataFile.setFileFormat(com.borland.dx.dataset.DataFileFormat.ASCII);

textDataFile.setSeparator("\t");
textDataFile.setLoadAsInserted(true);

try
{

/**
==================================================================*/
/** Removing first line (headers) */
/**
==================================================================*/
for (int i = 0; i < fileNames.length; i++)
{
String removerPath = filePath + "\\" + fileNames[i] + ".txt";
RandomAccessFile file = new RandomAccessFile(removerPath, "rws");
file.seek(0);
String line1 = file.readLine();
byte content[] = new byte[line1.length()];
file.seek(0);
file.write(content);
file.close();
}

// ==================================================================
// Importing
// ==================================================================
outputArea.append("Importing...:");
database1.openConnection();
for (int i = 0; i < fileNames.length; i++)
{

String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
textDataFile.setFileName(absolutePath);
textDataFile.load(tableDataSet1);
System.out.println("Current file: " + absolutePath);
outputArea.append("\nRows loaded: " + tableDataSet1.getRowCount());
database1.saveChanges(tableDataSet1);
tableDataSet1.clearValues();

}
database1.closeConnection();
outputArea.append("\nImport completed!");
}

catch (Exception ex)
{
ex.printStackTrace();
throw new Exception();
}

}

}


<---Code Snippet------------------------------------------------------>

Thanks,
Michael
--
=============================================
TeamB are volunteer helpers. Please DO NOT REPLY VIA EMAIL!
Post all questions and replies to this newsgroup ONLY
For papers on DataExpress, Applets, JSP, and Web Development go to:
www.microps.com/mps/paperFAQ.html
====================================================
 

Re:memory problems with DB import

Hi John
Yes I did. I used "XmX700" which gives java a heap size of up to this
amount. But thats crazy, I still get the out of memory error. I am
really flattered on how this can be...Even forcing a GC does not do
anything good...
Cheers,
Michael
John Moore wrote:
Quote
Michael,

Other than being a "very wide" table and therefore a LOT of column
objects.. Nothing is obviously off..

Have you tried giving the VM more memory..??

John..

Michael Boeni wrote:

>Dear all
>
>I have written an small tool that takes a tab-delimited text files and
>imports them into a PostgreSQL table. The import as such would run well,
>but I get a severe memory problem. The tool should import around 250
>text files, with the largest being 70 Megs.
>
>My problem is that the application does not seem to manage memory very
>well or I have introduced a memory leak somewhere. This snipped of code
>causes the problem:
>
><---Code Snippet------------------------------------------------------>
>outputArea.append("Importing...:");
>database1.openConnection();
>for (int i = 0; i < fileNames.length; i++)
>{
>
>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>textDataFile.setFileName(absolutePath);
>textDataFile.load(tableDataSet1);
>System.out.println("Current file: " + absolutePath);
>outputArea.append("\nRows loaded: " +
>tableDataSet1.getRowCount());
>database1.saveChanges(tableDataSet1);
>tableDataSet1.clearValues();
>
>}
>database1.closeConnection();
>outputArea.append("\nImport completed!");
>
><---Code Snippet------------------------------------------------------>
>
>Does anyone know what I am doing wrong here? How can I improve memory
>consumption?
>
>Here is the full class:
>
><---Code Snippet------------------------------------------------------>
>package com.shiftthink.geotools.data.geonames;
>
>import java.io.*;
>import com.borland.dx.dataset.*;
>import com.borland.dx.sql.dataset.*;
>import javax.swing.JTextArea;
>import javax.swing.JProgressBar;
>import java.util.Arrays;
>
>public class DBHelper
>{
>static String[] fileNames =
>{
>"ac", "al", "ag", "an", "ao", "av", "af", "ar", "am", "aa", "at",
>"as",
>"au", "aj", "bf", "ba", "bg", "bb", "bs", "bo", "be", "bh", "bn",
>"bd",
>"bt", "bl", "bk", "bc", "bv", "br", "io", "vi", "bx", "bu", "uv",
>"bm",
>"by", "cb", "cm", "ca", "cv", "cj", "ct", "cd", "ci", "ch", "kt",
>"ip",
>"ck", "co", "cn", "cf", "cg", "cw", "cr", "cs", "iv", "hr", "cu",
>"cy",
>"ez", "da", "dj", "do", "dr", "tt", "ec", "eg", "es", "ek", "er",
>"en",
>"et", "eu", "fk", "fo", "fj", "fi", "fr", "fg", "fp", "fs", "gb",
>"ga",
>"gz", "gg", "gm", "gh", "gi", "go", "gr", "gl", "gj", "gp", "gt",
>"gt",
>"gk", "pu", "gy", "ha", "hm", "ho", "hk", "hu", "ic", "in", "id",
>"ir",
>"iz", "ei", "im", "is", "it", "jm", "jn", "ja", "je", "jo", "ju",
>"kz",
>"ke", "kr", "ku", "kg", "la", "lg", "le", "lt", "li", "ly", "ls",
>"lh",
>"lu", "mc", "mk", "ma", "mi", "my", "mv", "ml", "mt", "rm", "mb",
>"mr",
>"mp", "mf", "mx", "fm", "md", "mn", "mg", "mh", "mo", "mz", "wa",
>"nr",
>"np", "nl", "nt", "nc", "nz", "nu", "ng", "ni", "ne", "nm", "nf",
>"kn",
>"no", "os", "mu", "pk", "ps", "pm", "pp", "pf", "pa", "pe", "rp",
>"pc",
>"pl", "po", "qa", "re", "ro", "rs", "rw", "sh", "sc", "st", "sb",
>"vc",
>"ws", "sm", "tp", "sa", "sg", "yi", "se", "sl", "sn", "lo", "si",
>"bp",
>"so", "sf", "sx", "ks", "sp", "pg", "ce", "su", "ns", "sv", "wz",
>"sw",
>"sz", "sy", "tw", "ti", "tz", "th", "to", "tl", "tn", "td", "te",
>"ts",
>"tu", "tx", "tk", "tv", "ug", "up", "uf", "ae", "uk", "uy", "uz",
>"nh",
>"vt", "ve", "vm", "wf", "we", "wi", "ym", "za", "zi"
>};
>TextDataFile textDataFile = new TextDataFile();
>TableDataSet tableDataSet1 = new TableDataSet();
>Column RC = new Column();
>Column UFI = new Column();
>Column UNI = new Column();
>Column LAT = new Column();
>Column LONG = new Column();
>Column DMS_LAT = new Column();
>Column DMS_LONG = new Column();
>Column UTM = new Column();
>Column JOG = new Column();
>Column FC = new Column();
>Column DSG = new Column();
>Column PC = new Column();
>Column CC1 = new Column();
>Column ADM1 = new Column();
>Column ADM2 = new Column();
>Column DIM = new Column();
>Column CC2 = new Column();
>Column NT = new Column();
>Column LC = new Column();
>Column SHORT_FORM = new Column();
>Column GENERIC = new Column();
>Column SHORT_NAME = new Column();
>Column FULL_NAME = new Column();
>Column FULL_NAME_ND = new Column();
>Column MOD_DATE = new Column();
>Database database1 = new Database();
>
>
>public DBHelper()
>{
>
>}
>
>
>public void gnImport(String filePath, JTextArea outputArea) throws
>Exception
>{
>database1.setConnection(new
>com.borland.dx.sql.dataset.ConnectionDescriptor("jdbc:postgresql:ST_EIS",
>"st_eis_user", "fairline", false, "org.postgresql.Driver"));
>MOD_DATE.setCaption("MOD_DATE");
>MOD_DATE.setColumnName("MOD_DATE");
>MOD_DATE.setDataType(com.borland.dx.dataset.Variant.STRING);
>MOD_DATE.setPreferredOrdinal(24);
>MOD_DATE.setWidth(15);
>MOD_DATE.setServerColumnName("MOD_DATE");
>MOD_DATE.setSqlType(0);
>FULL_NAME_ND.setCaption("FULL_NAME_ND");
>FULL_NAME_ND.setColumnName("FULL_NAME_ND");
>FULL_NAME_ND.setDataType(com.borland.dx.dataset.Variant.STRING);
>FULL_NAME_ND.setPreferredOrdinal(23);
>FULL_NAME_ND.setWidth(200);
>FULL_NAME_ND.setServerColumnName("FULL_NAME_ND");
>FULL_NAME_ND.setSqlType(0);
>FULL_NAME.setCaption("FULL_NAME");
>FULL_NAME.setColumnName("FULL_NAME");
>FULL_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>FULL_NAME.setPreferredOrdinal(22);
>FULL_NAME.setWidth(200);
>FULL_NAME.setServerColumnName("FULL_NAME");
>FULL_NAME.setSqlType(0);
>SHORT_NAME.setCaption("SHORT_NAME");
>SHORT_NAME.setColumnName("SHORT_NAME");
>SHORT_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>SHORT_NAME.setPreferredOrdinal(21);
>SHORT_NAME.setWidth(200);
>SHORT_NAME.setServerColumnName("SHORT_NAME");
>SHORT_NAME.setSqlType(0);
>GENERIC.setCaption("GENERIC");
>GENERIC.setColumnName("GENERIC");
>GENERIC.setDataType(com.borland.dx.dataset.Variant.STRING);
>GENERIC.setPreferredOrdinal(20);
>GENERIC.setWidth(128);
>GENERIC.setServerColumnName("GENERIC");
>GENERIC.setSqlType(0);
>SHORT_FORM.setCaption("SHORT_FORM");
>SHORT_FORM.setColumnName("SHORT_FORM");
>SHORT_FORM.setDataType(com.borland.dx.dataset.Variant.STRING);
>SHORT_FORM.setPreferredOrdinal(19);
>SHORT_FORM.setWidth(128);
>SHORT_FORM.setServerColumnName("SHORT_FORM");
>SHORT_FORM.setSqlType(0);
>LC.setCaption("LC");
>LC.setColumnName("LC");
>LC.setDataType(com.borland.dx.dataset.Variant.STRING);
>LC.setPreferredOrdinal(18);
>LC.setWidth(2);
>LC.setServerColumnName("LC");
>LC.setSqlType(0);
>NT.setCaption("NT");
>NT.setColumnName("NT");
>NT.setDataType(com.borland.dx.dataset.Variant.STRING);
>NT.setPreferredOrdinal(17);
>NT.setWidth(1);
>NT.setServerColumnName("NT");
>NT.setSqlType(0);
>CC2.setCaption("CC2");
>CC2.setColumnName("CC2");
>CC2.setDataType(com.borland.dx.dataset.Variant.STRING);
>CC2.setPreferredOrdinal(16);
>CC2.setWidth(3);
>CC2.setServerColumnName("CC2");
>CC2.setSqlType(0);
>DIM.setCaption("DIM");
>DIM.setColumnName("DIM");
>DIM.setDataType(com.borland.dx.dataset.Variant.STRING);
>DIM.setPreferredOrdinal(15);
>DIM.setWidth(15);
>DIM.setServerColumnName("DIM");
>DIM.setSqlType(0);
>ADM2.setCaption("ADM2");
>ADM2.setColumnName("ADM2");
>ADM2.setDataType(com.borland.dx.dataset.Variant.STRING);
>ADM2.setPreferredOrdinal(14);
>ADM2.setWidth(200);
>ADM2.setServerColumnName("ADM2");
>ADM2.setSqlType(0);
>ADM1.setCaption("ADM1");
>ADM1.setColumnName("ADM1");
>ADM1.setDataType(com.borland.dx.dataset.Variant.STRING);
>ADM1.setPreferredOrdinal(13);
>ADM1.setWidth(2);
>ADM1.setServerColumnName("ADM1");
>ADM1.setSqlType(0);
>CC1.setCaption("CC1");
>CC1.setColumnName("CC1");
>CC1.setDataType(com.borland.dx.dataset.Variant.STRING);
>CC1.setPreferredOrdinal(12);
>CC1.setWidth(3);
>CC1.setServerColumnName("CC1");
>CC1.setSqlType(0);
>PC.setCaption("PC");
>PC.setColumnName("PC");
>PC.setDataType(com.borland.dx.dataset.Variant.STRING);
>PC.setPreferredOrdinal(11);
>PC.setWidth(2);
>PC.setServerColumnName("PC");
>PC.setSqlType(0);
>DSG.setCaption("DSG");
>DSG.setColumnName("DSG");
>DSG.setDataType(com.borland.dx.dataset.Variant.STRING);
>DSG.setPreferredOrdinal(10);
>DSG.setWidth(12);
>DSG.setServerColumnName("DSG");
>DSG.setSqlType(0);
>FC.setCaption("FC");
>FC.setColumnName("FC");
>FC.setDataType(com.borland.dx.dataset.Variant.STRING);
>FC.setPreferredOrdinal(9);
>FC.setWidth(2);
>FC.setServerColumnName("FC");
>FC.setSqlType(0);
>JOG.setCaption("JOG");
>JOG.setColumnName("JOG");
>JOG.setDataType(com.borland.dx.dataset.Variant.STRING);
>JOG.setPreferredOrdinal(8);
>JOG.setWidth(10);
>JOG.setServerColumnName("JOG");
>JOG.setSqlType(0);
>UTM.setCaption("UTM");
>UTM.setColumnName("UTM");
>UTM.setDataType(com.borland.dx.dataset.Variant.STRING);
>UTM.setPreferredOrdinal(7);
>UTM.setWidth(5);
>UTM.setServerColumnName("UTM");
>UTM.setSqlType(0);
>DMS_LONG.setCaption("DMS_LONG");
>DMS_LONG.setColumnName("DMS_LONG");
>DMS_LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>DMS_LONG.setPreferredOrdinal(6);
>DMS_LONG.setServerColumnName("DMS_LONG");
>DMS_LONG.setWidth(15);
>DMS_LONG.setSqlType(0);
>DMS_LAT.setCaption("DMS_LAT");
>DMS_LAT.setColumnName("DMS_LAT");
>DMS_LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>DMS_LAT.setPreferredOrdinal(5);
>DMS_LAT.setServerColumnName("DMS_LAT");
>DMS_LAT.setWidth(15);
>DMS_LAT.setSqlType(0);
>LONG.setCaption("LONG");
>LONG.setColumnName("LONG");
>LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>LONG.setPreferredOrdinal(4);
>LONG.setServerColumnName("LONG");
>LONG.setWidth(15);
>LONG.setSqlType(0);
>LAT.setCaption("LAT");
>LAT.setColumnName("LAT");
>LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>LAT.setPreferredOrdinal(3);
>LAT.setServerColumnName("LAT");
>LAT.setWidth(15);
>LAT.setSqlType(0);
>UNI.setCaption("UNI");
>UNI.setColumnName("UNI");
>UNI.setDataType(com.borland.dx.dataset.Variant.STRING);
>UNI.setPreferredOrdinal(2);
>UNI.setServerColumnName("UNI");
>UNI.setWidth(15);
>UNI.setSqlType(0);
>UFI.setCaption("UFI");
>UFI.setColumnName("UFI");
>UFI.setDataType(com.borland.dx.dataset.Variant.STRING);
>UFI.setPreferredOrdinal(1);
>UFI.setServerColumnName("UFI");
>UFI.setWidth(15);
>UFI.setSqlType(0);
>RC.setCaption("RC");
>RC.setColumnName("RC");
>RC.setDataType(com.borland.dx.dataset.Variant.STRING);
>RC.setPreferredOrdinal(0);
>RC.setWidth(1);
>RC.setServerColumnName("RC");
>RC.setSqlType(0);
>
>
>tableDataSet1.setColumns(new Column[]
>{RC, UFI, UNI, LAT, LONG, DMS_LAT,
>DMS_LONG, UTM,
>JOG, FC, DSG, PC, CC1, ADM1, ADM2, DIM,
>CC2, NT,
>LC, SHORT_FORM, GENERIC, SHORT_NAME,
>FULL_NAME,
>FULL_NAME_ND, MOD_DATE});
>tableDataSet1.setTableName("coordinates");
>tableDataSet1.setSchemaName("st_eis");
>
>textDataFile.setDelimiter("");
>textDataFile.setEncoding("Cp1252");
>
>textDataFile.setFileFormat(com.borland.dx.dataset.DataFileFormat.ASCII);
>
>textDataFile.setSeparator("\t");
>textDataFile.setLoadAsInserted(true);
>
>try
>{
>
>/**
>==================================================================*/
>/** Removing first line (headers) */
>/**
>==================================================================*/
>for (int i = 0; i < fileNames.length; i++)
>{
>String removerPath = filePath + "\\" + fileNames[i] + ".txt";
>RandomAccessFile file = new RandomAccessFile(removerPath, "rws");
>file.seek(0);
>String line1 = file.readLine();
>byte content[] = new byte[line1.length()];
>file.seek(0);
>file.write(content);
>file.close();
>}
>
>//
>==================================================================
>// Importing
>//
>==================================================================
>outputArea.append("Importing...:");
>database1.openConnection();
>for (int i = 0; i < fileNames.length; i++)
>{
>
>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>textDataFile.setFileName(absolutePath);
>textDataFile.load(tableDataSet1);
>System.out.println("Current file: " + absolutePath);
>outputArea.append("\nRows loaded: " +
>tableDataSet1.getRowCount());
>database1.saveChanges(tableDataSet1);
>tableDataSet1.clearValues();
>
>}
>database1.closeConnection();
>outputArea.append("\nImport completed!");
>}
>
>catch (Exception ex)
>{
>ex.printStackTrace();
>throw new Exception();
>}
>
>}
>
>}
>
>
><---Code Snippet------------------------------------------------------>
>
>Thanks,
>Michael


 

{smallsort}

Re:memory problems with DB import

Well the next step would be to run this against OptimizeIt or some other
memory profiler so that you can narrow down were the "villian" is..
John..
Michael Boeni wrote:
Quote
Hi John

Yes I did. I used "XmX700" which gives java a heap size of up to this
amount. But thats crazy, I still get the out of memory error. I am
really flattered on how this can be...Even forcing a GC does not do
anything good...

Cheers,
Michael

John Moore wrote:

>Michael,
>
>Other than being a "very wide" table and therefore a LOT of column
>objects.. Nothing is obviously off..
>
>Have you tried giving the VM more memory..??
>
>John..
>
>Michael Boeni wrote:
>
>
>>Dear all
>>
>>I have written an small tool that takes a tab-delimited text files and
>>imports them into a PostgreSQL table. The import as such would run well,
>>but I get a severe memory problem. The tool should import around 250
>>text files, with the largest being 70 Megs.
>>
>>My problem is that the application does not seem to manage memory very
>>well or I have introduced a memory leak somewhere. This snipped of code
>>causes the problem:
>>
>><---Code Snippet------------------------------------------------------>
>>outputArea.append("Importing...:");
>>database1.openConnection();
>>for (int i = 0; i < fileNames.length; i++)
>>{
>>
>>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>>textDataFile.setFileName(absolutePath);
>>textDataFile.load(tableDataSet1);
>>System.out.println("Current file: " + absolutePath);
>>outputArea.append("\nRows loaded: " +
>>tableDataSet1.getRowCount());
>>database1.saveChanges(tableDataSet1);
>>tableDataSet1.clearValues();
>>
>>}
>>database1.closeConnection();
>>outputArea.append("\nImport completed!");
>>
>><---Code Snippet------------------------------------------------------>
>>
>>Does anyone know what I am doing wrong here? How can I improve memory
>>consumption?
>>
>>Here is the full class:
>>
>><---Code Snippet------------------------------------------------------>
>>package com.shiftthink.geotools.data.geonames;
>>
>>import java.io.*;
>>import com.borland.dx.dataset.*;
>>import com.borland.dx.sql.dataset.*;
>>import javax.swing.JTextArea;
>>import javax.swing.JProgressBar;
>>import java.util.Arrays;
>>
>>public class DBHelper
>>{
>>static String[] fileNames =
>>{
>>"ac", "al", "ag", "an", "ao", "av", "af", "ar", "am", "aa", "at",
>>"as",
>>"au", "aj", "bf", "ba", "bg", "bb", "bs", "bo", "be", "bh", "bn",
>>"bd",
>>"bt", "bl", "bk", "bc", "bv", "br", "io", "vi", "bx", "bu", "uv",
>>"bm",
>>"by", "cb", "cm", "ca", "cv", "cj", "ct", "cd", "ci", "ch", "kt",
>>"ip",
>>"ck", "co", "cn", "cf", "cg", "cw", "cr", "cs", "iv", "hr", "cu",
>>"cy",
>>"ez", "da", "dj", "do", "dr", "tt", "ec", "eg", "es", "ek", "er",
>>"en",
>>"et", "eu", "fk", "fo", "fj", "fi", "fr", "fg", "fp", "fs", "gb",
>>"ga",
>>"gz", "gg", "gm", "gh", "gi", "go", "gr", "gl", "gj", "gp", "gt",
>>"gt",
>>"gk", "pu", "gy", "ha", "hm", "ho", "hk", "hu", "ic", "in", "id",
>>"ir",
>>"iz", "ei", "im", "is", "it", "jm", "jn", "ja", "je", "jo", "ju",
>>"kz",
>>"ke", "kr", "ku", "kg", "la", "lg", "le", "lt", "li", "ly", "ls",
>>"lh",
>>"lu", "mc", "mk", "ma", "mi", "my", "mv", "ml", "mt", "rm", "mb",
>>"mr",
>>"mp", "mf", "mx", "fm", "md", "mn", "mg", "mh", "mo", "mz", "wa",
>>"nr",
>>"np", "nl", "nt", "nc", "nz", "nu", "ng", "ni", "ne", "nm", "nf",
>>"kn",
>>"no", "os", "mu", "pk", "ps", "pm", "pp", "pf", "pa", "pe", "rp",
>>"pc",
>>"pl", "po", "qa", "re", "ro", "rs", "rw", "sh", "sc", "st", "sb",
>>"vc",
>>"ws", "sm", "tp", "sa", "sg", "yi", "se", "sl", "sn", "lo", "si",
>>"bp",
>>"so", "sf", "sx", "ks", "sp", "pg", "ce", "su", "ns", "sv", "wz",
>>"sw",
>>"sz", "sy", "tw", "ti", "tz", "th", "to", "tl", "tn", "td", "te",
>>"ts",
>>"tu", "tx", "tk", "tv", "ug", "up", "uf", "ae", "uk", "uy", "uz",
>>"nh",
>>"vt", "ve", "vm", "wf", "we", "wi", "ym", "za", "zi"
>>};
>>TextDataFile textDataFile = new TextDataFile();
>>TableDataSet tableDataSet1 = new TableDataSet();
>>Column RC = new Column();
>>Column UFI = new Column();
>>Column UNI = new Column();
>>Column LAT = new Column();
>>Column LONG = new Column();
>>Column DMS_LAT = new Column();
>>Column DMS_LONG = new Column();
>>Column UTM = new Column();
>>Column JOG = new Column();
>>Column FC = new Column();
>>Column DSG = new Column();
>>Column PC = new Column();
>>Column CC1 = new Column();
>>Column ADM1 = new Column();
>>Column ADM2 = new Column();
>>Column DIM = new Column();
>>Column CC2 = new Column();
>>Column NT = new Column();
>>Column LC = new Column();
>>Column SHORT_FORM = new Column();
>>Column GENERIC = new Column();
>>Column SHORT_NAME = new Column();
>>Column FULL_NAME = new Column();
>>Column FULL_NAME_ND = new Column();
>>Column MOD_DATE = new Column();
>>Database database1 = new Database();
>>
>>
>>public DBHelper()
>>{
>>
>>}
>>
>>
>>public void gnImport(String filePath, JTextArea outputArea) throws
>>Exception
>>{
>>database1.setConnection(new
>>com.borland.dx.sql.dataset.ConnectionDescriptor("jdbc:postgresql:ST_EIS",
>>"st_eis_user", "fairline", false, "org.postgresql.Driver"));
>>MOD_DATE.setCaption("MOD_DATE");
>>MOD_DATE.setColumnName("MOD_DATE");
>>MOD_DATE.setDataType(com.borland.dx.dataset.Variant.STRING);
>>MOD_DATE.setPreferredOrdinal(24);
>>MOD_DATE.setWidth(15);
>>MOD_DATE.setServerColumnName("MOD_DATE");
>>MOD_DATE.setSqlType(0);
>>FULL_NAME_ND.setCaption("FULL_NAME_ND");
>>FULL_NAME_ND.setColumnName("FULL_NAME_ND");
>>FULL_NAME_ND.setDataType(com.borland.dx.dataset.Variant.STRING);
>>FULL_NAME_ND.setPreferredOrdinal(23);
>>FULL_NAME_ND.setWidth(200);
>>FULL_NAME_ND.setServerColumnName("FULL_NAME_ND");
>>FULL_NAME_ND.setSqlType(0);
>>FULL_NAME.setCaption("FULL_NAME");
>>FULL_NAME.setColumnName("FULL_NAME");
>>FULL_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>>FULL_NAME.setPreferredOrdinal(22);
>>FULL_NAME.setWidth(200);
>>FULL_NAME.setServerColumnName("FULL_NAME");
>>FULL_NAME.setSqlType(0);
>>SHORT_NAME.setCaption("SHORT_NAME");
>>SHORT_NAME.setColumnName("SHORT_NAME");
>>SHORT_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>>SHORT_NAME.setPreferredOrdinal(21);
>>SHORT_NAME.setWidth(200);
>>SHORT_NAME.setServerColumnName("SHORT_NAME");
>>SHORT_NAME.setSqlType(0);
>>GENERIC.setCaption("GENERIC");
>>GENERIC.setColumnName("GENERIC");
>>GENERIC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>GENERIC.setPreferredOrdinal(20);
>>GENERIC.setWidth(128);
>>GENERIC.setServerColumnName("GENERIC");
>>GENERIC.setSqlType(0);
>>SHORT_FORM.setCaption("SHORT_FORM");
>>SHORT_FORM.setColumnName("SHORT_FORM");
>>SHORT_FORM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>SHORT_FORM.setPreferredOrdinal(19);
>>SHORT_FORM.setWidth(128);
>>SHORT_FORM.setServerColumnName("SHORT_FORM");
>>SHORT_FORM.setSqlType(0);
>>LC.setCaption("LC");
>>LC.setColumnName("LC");
>>LC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>LC.setPreferredOrdinal(18);
>>LC.setWidth(2);
>>LC.setServerColumnName("LC");
>>LC.setSqlType(0);
>>NT.setCaption("NT");
>>NT.setColumnName("NT");
>>NT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>NT.setPreferredOrdinal(17);
>>NT.setWidth(1);
>>NT.setServerColumnName("NT");
>>NT.setSqlType(0);
>>CC2.setCaption("CC2");
>>CC2.setColumnName("CC2");
>>CC2.setDataType(com.borland.dx.dataset.Variant.STRING);
>>CC2.setPreferredOrdinal(16);
>>CC2.setWidth(3);
>>CC2.setServerColumnName("CC2");
>>CC2.setSqlType(0);
>>DIM.setCaption("DIM");
>>DIM.setColumnName("DIM");
>>DIM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>DIM.setPreferredOrdinal(15);
>>DIM.setWidth(15);
>>DIM.setServerColumnName("DIM");
>>DIM.setSqlType(0);
>>ADM2.setCaption("ADM2");
>>ADM2.setColumnName("ADM2");
>>ADM2.setDataType(com.borland.dx.dataset.Variant.STRING);
>>ADM2.setPreferredOrdinal(14);
>>ADM2.setWidth(200);
>>ADM2.setServerColumnName("ADM2");
>>ADM2.setSqlType(0);
>>ADM1.setCaption("ADM1");
>>ADM1.setColumnName("ADM1");
>>ADM1.setDataType(com.borland.dx.dataset.Variant.STRING);
>>ADM1.setPreferredOrdinal(13);
>>ADM1.setWidth(2);
>>ADM1.setServerColumnName("ADM1");
>>ADM1.setSqlType(0);
>>CC1.setCaption("CC1");
>>CC1.setColumnName("CC1");
>>CC1.setDataType(com.borland.dx.dataset.Variant.STRING);
>>CC1.setPreferredOrdinal(12);
>>CC1.setWidth(3);
>>CC1.setServerColumnName("CC1");
>>CC1.setSqlType(0);
>>PC.setCaption("PC");
>>PC.setColumnName("PC");
>>PC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>PC.setPreferredOrdinal(11);
>>PC.setWidth(2);
>>PC.setServerColumnName("PC");
>>PC.setSqlType(0);
>>DSG.setCaption("DSG");
>>DSG.setColumnName("DSG");
>>DSG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>DSG.setPreferredOrdinal(10);
>>DSG.setWidth(12);
>>DSG.setServerColumnName("DSG");
>>DSG.setSqlType(0);
>>FC.setCaption("FC");
>>FC.setColumnName("FC");
>>FC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>FC.setPreferredOrdinal(9);
>>FC.setWidth(2);
>>FC.setServerColumnName("FC");
>>FC.setSqlType(0);
>>JOG.setCaption("JOG");
>>JOG.setColumnName("JOG");
>>JOG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>JOG.setPreferredOrdinal(8);
>>JOG.setWidth(10);
>>JOG.setServerColumnName("JOG");
>>JOG.setSqlType(0);
>>UTM.setCaption("UTM");
>>UTM.setColumnName("UTM");
>>UTM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>UTM.setPreferredOrdinal(7);
>>UTM.setWidth(5);
>>UTM.setServerColumnName("UTM");
>>UTM.setSqlType(0);
>>DMS_LONG.setCaption("DMS_LONG");
>>DMS_LONG.setColumnName("DMS_LONG");
>>DMS_LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>DMS_LONG.setPreferredOrdinal(6);
>>DMS_LONG.setServerColumnName("DMS_LONG");
>>DMS_LONG.setWidth(15);
>>DMS_LONG.setSqlType(0);
>>DMS_LAT.setCaption("DMS_LAT");
>>DMS_LAT.setColumnName("DMS_LAT");
>>DMS_LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>DMS_LAT.setPreferredOrdinal(5);
>>DMS_LAT.setServerColumnName("DMS_LAT");
>>DMS_LAT.setWidth(15);
>>DMS_LAT.setSqlType(0);
>>LONG.setCaption("LONG");
>>LONG.setColumnName("LONG");
>>LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>LONG.setPreferredOrdinal(4);
>>LONG.setServerColumnName("LONG");
>>LONG.setWidth(15);
>>LONG.setSqlType(0);
>>LAT.setCaption("LAT");
>>LAT.setColumnName("LAT");
>>LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>LAT.setPreferredOrdinal(3);
>>LAT.setServerColumnName("LAT");
>>LAT.setWidth(15);
>>LAT.setSqlType(0);
>>UNI.setCaption("UNI");
>>UNI.setColumnName("UNI");
>>UNI.setDataType(com.borland.dx.dataset.Variant.STRING);
>>UNI.setPreferredOrdinal(2);
>>UNI.setServerColumnName("UNI");
>>UNI.setWidth(15);
>>UNI.setSqlType(0);
>>UFI.setCaption("UFI");
>>UFI.setColumnName("UFI");
>>UFI.setDataType(com.borland.dx.dataset.Variant.STRING);
>>UFI.setPreferredOrdinal(1);
>>UFI.setServerColumnName("UFI");
>>UFI.setWidth(15);
>>UFI.setSqlType(0);
>>RC.setCaption("RC");
>>RC.setColumnName("RC");
>>RC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>RC.setPreferredOrdinal(0);
>>RC.setWidth(1);
>>RC.setServerColumnName("RC");
>>RC.setSqlType(0);
>>
>>
>>tableDataSet1.setColumns(new Column[]
>>{RC, UFI, UNI, LAT, LONG, DMS_LAT,
>>DMS_LONG, UTM,
>>JOG, FC, DSG, PC, CC1, ADM1, ADM2, DIM,
>>CC2, NT,
>>LC, SHORT_FORM, GENERIC, SHORT_NAME,
>>FULL_NAME,
>>FULL_NAME_ND, MOD_DATE});
>>tableDataSet1.setTableName("coordinates");
>>tableDataSet1.setSchemaName("st_eis");
>>
>>textDataFile.setDelimiter("");
>>textDataFile.setEncoding("Cp1252");
>>
>>textDataFile.setFileFormat(com.borland.dx.dataset.DataFileFormat.ASCII);
>>
>>textDataFile.setSeparator("\t");
>>textDataFile.setLoadAsInserted(true);
>>
>>try
>>{
>>
>>/**
>>==================================================================*/
>>/** Removing first line (headers) */
>>/**
>>==================================================================*/
>>for (int i = 0; i < fileNames.length; i++)
>>{
>>String removerPath = filePath + "\\" + fileNames[i] + ".txt";
>>RandomAccessFile file = new RandomAccessFile(removerPath, "rws");
>>file.seek(0);
>>String line1 = file.readLine();
>>byte content[] = new byte[line1.length()];
>>file.seek(0);
>>file.write(content);
>>file.close();
>>}
>>
>>//
>>==================================================================
>>// Importing
>>//
>>==================================================================
>>outputArea.append("Importing...:");
>>database1.openConnection();
>>for (int i = 0; i < fileNames.length; i++)
>>{
>>
>>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>>textDataFile.setFileName(absolutePath);
>>textDataFile.load(tableDataSet1);
>>System.out.println("Current file: " + absolutePath);
>>outputArea.append("\nRows loaded: " +
>>tableDataSet1.getRowCount());
>>database1.saveChanges(tableDataSet1);
>>tableDataSet1.clearValues();
>>
>>}
>>database1.closeConnection();
>>outputArea.append("\nImport completed!");
>>}
>>
>>catch (Exception ex)
>>{
>>ex.printStackTrace();
>>throw new Exception();
>>}
>>
>>}
>>
>>}
>>
>>
>><---Code Snippet------------------------------------------------------>
>>
>>Thanks,
>>Michael
>
>
--
=============================================
TeamB are volunteer helpers. Please DO NOT REPLY VIA EMAIL!
Post all questions and replies to this newsgroup ONLY
For papers on DataExpress, Applets, JSP, and Web Development go to:
www.microps.com/mps/paperFAQ.html
====================================================
 

Re:memory problems with DB import

Hi John
I did that ant get strange results. According to Optimizeit, the Heap is
regularily emptied and then memory consumption mounts again - which is
as I expected. The strange thing is that the memory consumed as shown by
Windows only goes upward, until Win starts swapping...
I cannot believe that the DataExpress components are so extremely
memory-inefficient. There must be a way to improve that...
John Moore wrote:
Quote
Well the next step would be to run this against OptimizeIt or some other
memory profiler so that you can narrow down were the "villian" is..

John..

Michael Boeni wrote:

>Hi John
>
>Yes I did. I used "XmX700" which gives java a heap size of up to this
>amount. But thats crazy, I still get the out of memory error. I am
>really flattered on how this can be...Even forcing a GC does not do
>anything good...
>
>Cheers,
>Michael
>
>John Moore wrote:
>
>>Michael,
>>
>>Other than being a "very wide" table and therefore a LOT of column
>>objects.. Nothing is obviously off..
>>
>>Have you tried giving the VM more memory..??
>>
>>John..
>>
>>Michael Boeni wrote:
>>
>>
>>>Dear all
>>>
>>>I have written an small tool that takes a tab-delimited text files and
>>>imports them into a PostgreSQL table. The import as such would run
>>>well,
>>>but I get a severe memory problem. The tool should import around 250
>>>text files, with the largest being 70 Megs.
>>>
>>>My problem is that the application does not seem to manage memory very
>>>well or I have introduced a memory leak somewhere. This snipped of code
>>>causes the problem:
>>>
>>><---Code Snippet------------------------------------------------------>
>>>outputArea.append("Importing...:");
>>>database1.openConnection();
>>>for (int i = 0; i < fileNames.length; i++)
>>>{
>>>
>>>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>>>textDataFile.setFileName(absolutePath);
>>>textDataFile.load(tableDataSet1);
>>>System.out.println("Current file: " + absolutePath);
>>>outputArea.append("\nRows loaded: " +
>>>tableDataSet1.getRowCount());
>>>database1.saveChanges(tableDataSet1);
>>>tableDataSet1.clearValues();
>>>
>>>}
>>>database1.closeConnection();
>>>outputArea.append("\nImport completed!");
>>>
>>><---Code Snippet------------------------------------------------------>
>>>
>>>Does anyone know what I am doing wrong here? How can I improve memory
>>>consumption?
>>>
>>>Here is the full class:
>>>
>>><---Code Snippet------------------------------------------------------>
>>>package com.shiftthink.geotools.data.geonames;
>>>
>>>import java.io.*;
>>>import com.borland.dx.dataset.*;
>>>import com.borland.dx.sql.dataset.*;
>>>import javax.swing.JTextArea;
>>>import javax.swing.JProgressBar;
>>>import java.util.Arrays;
>>>
>>>public class DBHelper
>>>{
>>>static String[] fileNames =
>>>{
>>>"ac", "al", "ag", "an", "ao", "av", "af", "ar", "am", "aa", "at",
>>>"as",
>>>"au", "aj", "bf", "ba", "bg", "bb", "bs", "bo", "be", "bh", "bn",
>>>"bd",
>>>"bt", "bl", "bk", "bc", "bv", "br", "io", "vi", "bx", "bu", "uv",
>>>"bm",
>>>"by", "cb", "cm", "ca", "cv", "cj", "ct", "cd", "ci", "ch", "kt",
>>>"ip",
>>>"ck", "co", "cn", "cf", "cg", "cw", "cr", "cs", "iv", "hr", "cu",
>>>"cy",
>>>"ez", "da", "dj", "do", "dr", "tt", "ec", "eg", "es", "ek", "er",
>>>"en",
>>>"et", "eu", "fk", "fo", "fj", "fi", "fr", "fg", "fp", "fs", "gb",
>>>"ga",
>>>"gz", "gg", "gm", "gh", "gi", "go", "gr", "gl", "gj", "gp", "gt",
>>>"gt",
>>>"gk", "pu", "gy", "ha", "hm", "ho", "hk", "hu", "ic", "in", "id",
>>>"ir",
>>>"iz", "ei", "im", "is", "it", "jm", "jn", "ja", "je", "jo", "ju",
>>>"kz",
>>>"ke", "kr", "ku", "kg", "la", "lg", "le", "lt", "li", "ly", "ls",
>>>"lh",
>>>"lu", "mc", "mk", "ma", "mi", "my", "mv", "ml", "mt", "rm", "mb",
>>>"mr",
>>>"mp", "mf", "mx", "fm", "md", "mn", "mg", "mh", "mo", "mz", "wa",
>>>"nr",
>>>"np", "nl", "nt", "nc", "nz", "nu", "ng", "ni", "ne", "nm", "nf",
>>>"kn",
>>>"no", "os", "mu", "pk", "ps", "pm", "pp", "pf", "pa", "pe", "rp",
>>>"pc",
>>>"pl", "po", "qa", "re", "ro", "rs", "rw", "sh", "sc", "st", "sb",
>>>"vc",
>>>"ws", "sm", "tp", "sa", "sg", "yi", "se", "sl", "sn", "lo", "si",
>>>"bp",
>>>"so", "sf", "sx", "ks", "sp", "pg", "ce", "su", "ns", "sv", "wz",
>>>"sw",
>>>"sz", "sy", "tw", "ti", "tz", "th", "to", "tl", "tn", "td", "te",
>>>"ts",
>>>"tu", "tx", "tk", "tv", "ug", "up", "uf", "ae", "uk", "uy", "uz",
>>>"nh",
>>>"vt", "ve", "vm", "wf", "we", "wi", "ym", "za", "zi"
>>>};
>>>TextDataFile textDataFile = new TextDataFile();
>>>TableDataSet tableDataSet1 = new TableDataSet();
>>>Column RC = new Column();
>>>Column UFI = new Column();
>>>Column UNI = new Column();
>>>Column LAT = new Column();
>>>Column LONG = new Column();
>>>Column DMS_LAT = new Column();
>>>Column DMS_LONG = new Column();
>>>Column UTM = new Column();
>>>Column JOG = new Column();
>>>Column FC = new Column();
>>>Column DSG = new Column();
>>>Column PC = new Column();
>>>Column CC1 = new Column();
>>>Column ADM1 = new Column();
>>>Column ADM2 = new Column();
>>>Column DIM = new Column();
>>>Column CC2 = new Column();
>>>Column NT = new Column();
>>>Column LC = new Column();
>>>Column SHORT_FORM = new Column();
>>>Column GENERIC = new Column();
>>>Column SHORT_NAME = new Column();
>>>Column FULL_NAME = new Column();
>>>Column FULL_NAME_ND = new Column();
>>>Column MOD_DATE = new Column();
>>>Database database1 = new Database();
>>>
>>>
>>>public DBHelper()
>>>{
>>>
>>>}
>>>
>>>
>>>public void gnImport(String filePath, JTextArea outputArea) throws
>>>Exception
>>>{
>>>database1.setConnection(new
>>>com.borland.dx.sql.dataset.ConnectionDescriptor("jdbc:postgresql:ST_EIS",
>>>
>>>"st_eis_user", "fairline", false, "org.postgresql.Driver"));
>>>MOD_DATE.setCaption("MOD_DATE");
>>>MOD_DATE.setColumnName("MOD_DATE");
>>>MOD_DATE.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>MOD_DATE.setPreferredOrdinal(24);
>>>MOD_DATE.setWidth(15);
>>>MOD_DATE.setServerColumnName("MOD_DATE");
>>>MOD_DATE.setSqlType(0);
>>>FULL_NAME_ND.setCaption("FULL_NAME_ND");
>>>FULL_NAME_ND.setColumnName("FULL_NAME_ND");
>>>FULL_NAME_ND.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>FULL_NAME_ND.setPreferredOrdinal(23);
>>>FULL_NAME_ND.setWidth(200);
>>>FULL_NAME_ND.setServerColumnName("FULL_NAME_ND");
>>>FULL_NAME_ND.setSqlType(0);
>>>FULL_NAME.setCaption("FULL_NAME");
>>>FULL_NAME.setColumnName("FULL_NAME");
>>>FULL_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>FULL_NAME.setPreferredOrdinal(22);
>>>FULL_NAME.setWidth(200);
>>>FULL_NAME.setServerColumnName("FULL_NAME");
>>>FULL_NAME.setSqlType(0);
>>>SHORT_NAME.setCaption("SHORT_NAME");
>>>SHORT_NAME.setColumnName("SHORT_NAME");
>>>SHORT_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>SHORT_NAME.setPreferredOrdinal(21);
>>>SHORT_NAME.setWidth(200);
>>>SHORT_NAME.setServerColumnName("SHORT_NAME");
>>>SHORT_NAME.setSqlType(0);
>>>GENERIC.setCaption("GENERIC");
>>>GENERIC.setColumnName("GENERIC");
>>>GENERIC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>GENERIC.setPreferredOrdinal(20);
>>>GENERIC.setWidth(128);
>>>GENERIC.setServerColumnName("GENERIC");
>>>GENERIC.setSqlType(0);
>>>SHORT_FORM.setCaption("SHORT_FORM");
>>>SHORT_FORM.setColumnName("SHORT_FORM");
>>>SHORT_FORM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>SHORT_FORM.setPreferredOrdinal(19);
>>>SHORT_FORM.setWidth(128);
>>>SHORT_FORM.setServerColumnName("SHORT_FORM");
>>>SHORT_FORM.setSqlType(0);
>>>LC.setCaption("LC");
>>>LC.setColumnName("LC");
>>>LC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>LC.setPreferredOrdinal(18);
>>>LC.setWidth(2);
>>>LC.setServerColumnName("LC");
>>>LC.setSqlType(0);
>>>NT.setCaption("NT");
>>>NT.setColumnName("NT");
>>>NT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>NT.setPreferredOrdinal(17);
>>>NT.setWidth(1);
>>>NT.setServerColumnName("NT");
>>>NT.setSqlType(0);
>>>CC2.setCaption("CC2");
>>>CC2.setColumnName("CC2");
>>>CC2.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>CC2.setPreferredOrdinal(16);
>>>CC2.setWidth(3);
>>>CC2.setServerColumnName("CC2");
>>>CC2.setSqlType(0);
>>>DIM.setCaption("DIM");
>>>DIM.setColumnName("DIM");
>>>DIM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>DIM.setPreferredOrdinal(15);
>>>DIM.setWidth(15);
>>>DIM.setServerColumnName("DIM");
>>>DIM.setSqlType(0);
>>>ADM2.setCaption("ADM2");
>>>ADM2.setColumnName("ADM2");
>>>ADM2.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>ADM2.setPreferredOrdinal(14);
>>>ADM2.setWidth(200);
>>>ADM2.setServerColumnName("ADM2");
>>>ADM2.setSqlType(0);
>>>ADM1.setCaption("ADM1");
>>>ADM1.setColumnName("ADM1");
>>>ADM1.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>ADM1.setPreferredOrdinal(13);
>>>ADM1.setWidth(2);
>>>ADM1.setServerColumnName("ADM1");
>>>ADM1.setSqlType(0);
>>>CC1.setCaption("CC1");
>>>CC1.setColumnName("CC1");
>>>CC1.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>CC1.setPreferredOrdinal(12);
>>>CC1.setWidth(3);
>>>CC1.setServerColumnName("CC1");
>>>CC1.setSqlType(0);
>>>PC.setCaption("PC");
>>>PC.setColumnName("PC");
>>>PC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>PC.setPreferredOrdinal(11);
>>>PC.setWidth(2);
>>>PC.setServerColumnName("PC");
>>>PC.setSqlType(0);
>>>DSG.setCaption("DSG");
>>>DSG.setColumnName("DSG");
>>>DSG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>DSG.setPreferredOrdinal(10);
>>>DSG.setWidth(12);
>>>DSG.setServerColumnName("DSG");
>>>DSG.setSqlType(0);
>>>FC.setCaption("FC");
>>>FC.setColumnName("FC");
>>>FC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>FC.setPreferredOrdinal(9);
>>>FC.setWidth(2);
>>>FC.setServerColumnName("FC");
>>>FC.setSqlType(0);
>>>JOG.setCaption("JOG");
>>>JOG.setColumnName("JOG");
>>>JOG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>JOG.setPreferredOrdinal(8);
>>>JOG.setWidth(10);
>>>JOG.setServerColumnName("JOG");
>>>JOG.setSqlType(0);
>>>UTM.setCaption("UTM");
>>>UTM.setColumnName("UTM");
>>>UTM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>UTM.setPreferredOrdinal(7);
>>>UTM.setWidth(5);
>>>UTM.setServerColumnName("UTM");
>>>UTM.setSqlType(0);
>>>DMS_LONG.setCaption("DMS_LONG");
>>>DMS_LONG.setColumnName("DMS_LONG");
>>>DMS_LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>DMS_LONG.setPreferredOrdinal(6);
>>>DMS_LONG.setServerColumnName("DMS_LONG");
>>>DMS_LONG.setWidth(15);
>>>DMS_LONG.setSqlType(0);
>>>DMS_LAT.setCaption("DMS_LAT");
>>>DMS_LAT.setColumnName("DMS_LAT");
>>>DMS_LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>DMS_LAT.setPreferredOrdinal(5);
>>>DMS_LAT.setServerColumnName("DMS_LAT");
>>>DMS_LAT.setWidth(15);
>>>DMS_LAT.setSqlType(0);
>>>LONG.setCaption("LONG");
>>>LONG.setColumnName("LONG");
>>>LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>LONG.setPreferredOrdinal(4);
>>>LONG.setServerColumnName("LONG");
>>>LONG.setWidth(15);
>>>LONG.setSqlType(0);
>>>LAT.setCaption("LAT");
>>>LAT.setColumnName("LAT");
>>>LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>LAT.setPreferredOrdinal(3);
>>>LAT.setServerColumnName("LAT");
>>>LAT.setWidth(15);
>>>LAT.setSqlType(0);
>>>UNI.setCaption("UNI");
>>>UNI.setColumnName("UNI");
>>>UNI.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>UNI.setPreferredOrdinal(2);
>>>UNI.setServerColumnName("UNI");
>>>UNI.setWidth(15);
>>>UNI.setSqlType(0);
>>>UFI.setCaption("UFI");
>>>UFI.setColumnName("UFI");
>>>UFI.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>UFI.setPreferredOrdinal(1);
>>>UFI.setServerColumnName("UFI");
>>>UFI.setWidth(15);
>>>UFI.setSqlType(0);
>>>RC.setCaption("RC");
>>>RC.setColumnName("RC");
>>>RC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>RC.setPreferredOrdinal(0);
>>>RC.setWidth(1);
>>>RC.setServerColumnName("RC");
>>>RC.setSqlType(0);
>>>
>>>
>>>tableDataSet1.setColumns(new Column[]
>>>{RC, UFI, UNI, LAT, LONG, DMS_LAT,
>>>DMS_LONG, UTM,
>>>JOG, FC, DSG, PC, CC1, ADM1, ADM2, DIM,
>>>CC2, NT,
>>>LC, SHORT_FORM, GENERIC, SHORT_NAME,
>>>FULL_NAME,
>>>FULL_NAME_ND, MOD_DATE});
>>>tableDataSet1.setTableName("coordinates");
>>>tableDataSet1.setSchemaName("st_eis");
>>>
>>>textDataFile.setDelimiter("");
>>>textDataFile.setEncoding("Cp1252");
>>>
>>>textDataFile.setFileFormat(com.borland.dx.dataset.DataFileFormat.ASCII);
>>>
>>>
>>>textDataFile.setSeparator("\t");
>>>textDataFile.setLoadAsInserted(true);
>>>
>>>try
>>>{
>>>
>>>/**
>>>==================================================================*/
>>>/** Removing first line (headers) */
>>>/**
>>>==================================================================*/
>>>for (int i = 0; i < fileNames.length; i++)
>>>{
>>>String removerPath = filePath + "\\" + fileNames[i] + ".txt";
>>>RandomAccessFile file = new RandomAccessFile(removerPath,
>>>"rws");
>>>file.seek(0);
>>>String line1 = file.readLine();
>>>byte content[] = new byte[line1.length()];
>>>file.seek(0);
>>>file.write(content);
>>>file.close();
>>>}
>>>
>>>//
>>>==================================================================
>>>// Importing
>>>//
>>>==================================================================
>>>outputArea.append("Importing...:");
>>>database1.openConnection();
>>>for (int i = 0; i < fileNames.length; i++)
>>>{
>>>
>>>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>>>textDataFile.setFileName(absolutePath);
>>>textDataFile.load(tableDataSet1);
>>>System.out.println("Current file: " + absolutePath);
>>>outputArea.append("\nRows loaded: " +
>>>tableDataSet1.getRowCount());
>>>database1.saveChanges(tableDataSet1);
>>>tableDataSet1.clearValues();
>>>
>>>}
>>>database1.closeConnection();
>>>outputArea.append("\nImport completed!");
>>>}
>>>
>>>catch (Exception ex)
>>>{
>>>ex.printStackTrace();
>>>throw new Exception();
>>>}
>>>
>>>}
>>>
>>>}
>>>
>>>
>>><---Code Snippet------------------------------------------------------>
>>>
>>>Thanks,
>>>Michael
>>
>>
>>

 

Re:memory problems with DB import

If you can...
Produce a "small" demo that trips this problem.. Provide repeatable
instructions..
email it to me to verify..
If I can duplicate it, I will pass it on to the "right" persons..
John..
Michael Boeni wrote:
Quote
Hi John

I did that ant get strange results. According to Optimizeit, the Heap is
regularily emptied and then memory consumption mounts again - which is
as I expected. The strange thing is that the memory consumed as shown by
Windows only goes upward, until Win starts swapping...

I cannot believe that the DataExpress components are so extremely
memory-inefficient. There must be a way to improve that...



John Moore wrote:

>Well the next step would be to run this against OptimizeIt or some other
>memory profiler so that you can narrow down were the "villian" is..
>
>John..
>
>Michael Boeni wrote:
>
>
>>Hi John
>>
>>Yes I did. I used "XmX700" which gives java a heap size of up to this
>>amount. But thats crazy, I still get the out of memory error. I am
>>really flattered on how this can be...Even forcing a GC does not do
>>anything good...
>>
>>Cheers,
>>Michael
>>
>>John Moore wrote:
>>
>>
>>>Michael,
>>>
>>>Other than being a "very wide" table and therefore a LOT of column
>>>objects.. Nothing is obviously off..
>>>
>>>Have you tried giving the VM more memory..??
>>>
>>>John..
>>>
>>>Michael Boeni wrote:
>>>
>>>
>>>
>>>>Dear all
>>>>
>>>>I have written an small tool that takes a tab-delimited text files and
>>>>imports them into a PostgreSQL table. The import as such would run
>>>>well,
>>>>but I get a severe memory problem. The tool should import around 250
>>>>text files, with the largest being 70 Megs.
>>>>
>>>>My problem is that the application does not seem to manage memory very
>>>>well or I have introduced a memory leak somewhere. This snipped of code
>>>>causes the problem:
>>>>
>>>><---Code Snippet------------------------------------------------------>
>>>>outputArea.append("Importing...:");
>>>>database1.openConnection();
>>>>for (int i = 0; i < fileNames.length; i++)
>>>>{
>>>>
>>>>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>>>>textDataFile.setFileName(absolutePath);
>>>>textDataFile.load(tableDataSet1);
>>>>System.out.println("Current file: " + absolutePath);
>>>>outputArea.append("\nRows loaded: " +
>>>>tableDataSet1.getRowCount());
>>>>database1.saveChanges(tableDataSet1);
>>>>tableDataSet1.clearValues();
>>>>
>>>>}
>>>>database1.closeConnection();
>>>>outputArea.append("\nImport completed!");
>>>>
>>>><---Code Snippet------------------------------------------------------>
>>>>
>>>>Does anyone know what I am doing wrong here? How can I improve memory
>>>>consumption?
>>>>
>>>>Here is the full class:
>>>>
>>>><---Code Snippet------------------------------------------------------>
>>>>package com.shiftthink.geotools.data.geonames;
>>>>
>>>>import java.io.*;
>>>>import com.borland.dx.dataset.*;
>>>>import com.borland.dx.sql.dataset.*;
>>>>import javax.swing.JTextArea;
>>>>import javax.swing.JProgressBar;
>>>>import java.util.Arrays;
>>>>
>>>>public class DBHelper
>>>>{
>>>>static String[] fileNames =
>>>>{
>>>>"ac", "al", "ag", "an", "ao", "av", "af", "ar", "am", "aa", "at",
>>>>"as",
>>>>"au", "aj", "bf", "ba", "bg", "bb", "bs", "bo", "be", "bh", "bn",
>>>>"bd",
>>>>"bt", "bl", "bk", "bc", "bv", "br", "io", "vi", "bx", "bu", "uv",
>>>>"bm",
>>>>"by", "cb", "cm", "ca", "cv", "cj", "ct", "cd", "ci", "ch", "kt",
>>>>"ip",
>>>>"ck", "co", "cn", "cf", "cg", "cw", "cr", "cs", "iv", "hr", "cu",
>>>>"cy",
>>>>"ez", "da", "dj", "do", "dr", "tt", "ec", "eg", "es", "ek", "er",
>>>>"en",
>>>>"et", "eu", "fk", "fo", "fj", "fi", "fr", "fg", "fp", "fs", "gb",
>>>>"ga",
>>>>"gz", "gg", "gm", "gh", "gi", "go", "gr", "gl", "gj", "gp", "gt",
>>>>"gt",
>>>>"gk", "pu", "gy", "ha", "hm", "ho", "hk", "hu", "ic", "in", "id",
>>>>"ir",
>>>>"iz", "ei", "im", "is", "it", "jm", "jn", "ja", "je", "jo", "ju",
>>>>"kz",
>>>>"ke", "kr", "ku", "kg", "la", "lg", "le", "lt", "li", "ly", "ls",
>>>>"lh",
>>>>"lu", "mc", "mk", "ma", "mi", "my", "mv", "ml", "mt", "rm", "mb",
>>>>"mr",
>>>>"mp", "mf", "mx", "fm", "md", "mn", "mg", "mh", "mo", "mz", "wa",
>>>>"nr",
>>>>"np", "nl", "nt", "nc", "nz", "nu", "ng", "ni", "ne", "nm", "nf",
>>>>"kn",
>>>>"no", "os", "mu", "pk", "ps", "pm", "pp", "pf", "pa", "pe", "rp",
>>>>"pc",
>>>>"pl", "po", "qa", "re", "ro", "rs", "rw", "sh", "sc", "st", "sb",
>>>>"vc",
>>>>"ws", "sm", "tp", "sa", "sg", "yi", "se", "sl", "sn", "lo", "si",
>>>>"bp",
>>>>"so", "sf", "sx", "ks", "sp", "pg", "ce", "su", "ns", "sv", "wz",
>>>>"sw",
>>>>"sz", "sy", "tw", "ti", "tz", "th", "to", "tl", "tn", "td", "te",
>>>>"ts",
>>>>"tu", "tx", "tk", "tv", "ug", "up", "uf", "ae", "uk", "uy", "uz",
>>>>"nh",
>>>>"vt", "ve", "vm", "wf", "we", "wi", "ym", "za", "zi"
>>>>};
>>>>TextDataFile textDataFile = new TextDataFile();
>>>>TableDataSet tableDataSet1 = new TableDataSet();
>>>>Column RC = new Column();
>>>>Column UFI = new Column();
>>>>Column UNI = new Column();
>>>>Column LAT = new Column();
>>>>Column LONG = new Column();
>>>>Column DMS_LAT = new Column();
>>>>Column DMS_LONG = new Column();
>>>>Column UTM = new Column();
>>>>Column JOG = new Column();
>>>>Column FC = new Column();
>>>>Column DSG = new Column();
>>>>Column PC = new Column();
>>>>Column CC1 = new Column();
>>>>Column ADM1 = new Column();
>>>>Column ADM2 = new Column();
>>>>Column DIM = new Column();
>>>>Column CC2 = new Column();
>>>>Column NT = new Column();
>>>>Column LC = new Column();
>>>>Column SHORT_FORM = new Column();
>>>>Column GENERIC = new Column();
>>>>Column SHORT_NAME = new Column();
>>>>Column FULL_NAME = new Column();
>>>>Column FULL_NAME_ND = new Column();
>>>>Column MOD_DATE = new Column();
>>>>Database database1 = new Database();
>>>>
>>>>
>>>>public DBHelper()
>>>>{
>>>>
>>>>}
>>>>
>>>>
>>>>public void gnImport(String filePath, JTextArea outputArea) throws
>>>>Exception
>>>>{
>>>>database1.setConnection(new
>>>>com.borland.dx.sql.dataset.ConnectionDescriptor("jdbc:postgresql:ST_EIS",
>>>>
>>>>"st_eis_user", "fairline", false, "org.postgresql.Driver"));
>>>>MOD_DATE.setCaption("MOD_DATE");
>>>>MOD_DATE.setColumnName("MOD_DATE");
>>>>MOD_DATE.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>MOD_DATE.setPreferredOrdinal(24);
>>>>MOD_DATE.setWidth(15);
>>>>MOD_DATE.setServerColumnName("MOD_DATE");
>>>>MOD_DATE.setSqlType(0);
>>>>FULL_NAME_ND.setCaption("FULL_NAME_ND");
>>>>FULL_NAME_ND.setColumnName("FULL_NAME_ND");
>>>>FULL_NAME_ND.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>FULL_NAME_ND.setPreferredOrdinal(23);
>>>>FULL_NAME_ND.setWidth(200);
>>>>FULL_NAME_ND.setServerColumnName("FULL_NAME_ND");
>>>>FULL_NAME_ND.setSqlType(0);
>>>>FULL_NAME.setCaption("FULL_NAME");
>>>>FULL_NAME.setColumnName("FULL_NAME");
>>>>FULL_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>FULL_NAME.setPreferredOrdinal(22);
>>>>FULL_NAME.setWidth(200);
>>>>FULL_NAME.setServerColumnName("FULL_NAME");
>>>>FULL_NAME.setSqlType(0);
>>>>SHORT_NAME.setCaption("SHORT_NAME");
>>>>SHORT_NAME.setColumnName("SHORT_NAME");
>>>>SHORT_NAME.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>SHORT_NAME.setPreferredOrdinal(21);
>>>>SHORT_NAME.setWidth(200);
>>>>SHORT_NAME.setServerColumnName("SHORT_NAME");
>>>>SHORT_NAME.setSqlType(0);
>>>>GENERIC.setCaption("GENERIC");
>>>>GENERIC.setColumnName("GENERIC");
>>>>GENERIC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>GENERIC.setPreferredOrdinal(20);
>>>>GENERIC.setWidth(128);
>>>>GENERIC.setServerColumnName("GENERIC");
>>>>GENERIC.setSqlType(0);
>>>>SHORT_FORM.setCaption("SHORT_FORM");
>>>>SHORT_FORM.setColumnName("SHORT_FORM");
>>>>SHORT_FORM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>SHORT_FORM.setPreferredOrdinal(19);
>>>>SHORT_FORM.setWidth(128);
>>>>SHORT_FORM.setServerColumnName("SHORT_FORM");
>>>>SHORT_FORM.setSqlType(0);
>>>>LC.setCaption("LC");
>>>>LC.setColumnName("LC");
>>>>LC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>LC.setPreferredOrdinal(18);
>>>>LC.setWidth(2);
>>>>LC.setServerColumnName("LC");
>>>>LC.setSqlType(0);
>>>>NT.setCaption("NT");
>>>>NT.setColumnName("NT");
>>>>NT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>NT.setPreferredOrdinal(17);
>>>>NT.setWidth(1);
>>>>NT.setServerColumnName("NT");
>>>>NT.setSqlType(0);
>>>>CC2.setCaption("CC2");
>>>>CC2.setColumnName("CC2");
>>>>CC2.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>CC2.setPreferredOrdinal(16);
>>>>CC2.setWidth(3);
>>>>CC2.setServerColumnName("CC2");
>>>>CC2.setSqlType(0);
>>>>DIM.setCaption("DIM");
>>>>DIM.setColumnName("DIM");
>>>>DIM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>DIM.setPreferredOrdinal(15);
>>>>DIM.setWidth(15);
>>>>DIM.setServerColumnName("DIM");
>>>>DIM.setSqlType(0);
>>>>ADM2.setCaption("ADM2");
>>>>ADM2.setColumnName("ADM2");
>>>>ADM2.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>ADM2.setPreferredOrdinal(14);
>>>>ADM2.setWidth(200);
>>>>ADM2.setServerColumnName("ADM2");
>>>>ADM2.setSqlType(0);
>>>>ADM1.setCaption("ADM1");
>>>>ADM1.setColumnName("ADM1");
>>>>ADM1.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>ADM1.setPreferredOrdinal(13);
>>>>ADM1.setWidth(2);
>>>>ADM1.setServerColumnName("ADM1");
>>>>ADM1.setSqlType(0);
>>>>CC1.setCaption("CC1");
>>>>CC1.setColumnName("CC1");
>>>>CC1.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>CC1.setPreferredOrdinal(12);
>>>>CC1.setWidth(3);
>>>>CC1.setServerColumnName("CC1");
>>>>CC1.setSqlType(0);
>>>>PC.setCaption("PC");
>>>>PC.setColumnName("PC");
>>>>PC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>PC.setPreferredOrdinal(11);
>>>>PC.setWidth(2);
>>>>PC.setServerColumnName("PC");
>>>>PC.setSqlType(0);
>>>>DSG.setCaption("DSG");
>>>>DSG.setColumnName("DSG");
>>>>DSG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>DSG.setPreferredOrdinal(10);
>>>>DSG.setWidth(12);
>>>>DSG.setServerColumnName("DSG");
>>>>DSG.setSqlType(0);
>>>>FC.setCaption("FC");
>>>>FC.setColumnName("FC");
>>>>FC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>FC.setPreferredOrdinal(9);
>>>>FC.setWidth(2);
>>>>FC.setServerColumnName("FC");
>>>>FC.setSqlType(0);
>>>>JOG.setCaption("JOG");
>>>>JOG.setColumnName("JOG");
>>>>JOG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>JOG.setPreferredOrdinal(8);
>>>>JOG.setWidth(10);
>>>>JOG.setServerColumnName("JOG");
>>>>JOG.setSqlType(0);
>>>>UTM.setCaption("UTM");
>>>>UTM.setColumnName("UTM");
>>>>UTM.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>UTM.setPreferredOrdinal(7);
>>>>UTM.setWidth(5);
>>>>UTM.setServerColumnName("UTM");
>>>>UTM.setSqlType(0);
>>>>DMS_LONG.setCaption("DMS_LONG");
>>>>DMS_LONG.setColumnName("DMS_LONG");
>>>>DMS_LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>DMS_LONG.setPreferredOrdinal(6);
>>>>DMS_LONG.setServerColumnName("DMS_LONG");
>>>>DMS_LONG.setWidth(15);
>>>>DMS_LONG.setSqlType(0);
>>>>DMS_LAT.setCaption("DMS_LAT");
>>>>DMS_LAT.setColumnName("DMS_LAT");
>>>>DMS_LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>DMS_LAT.setPreferredOrdinal(5);
>>>>DMS_LAT.setServerColumnName("DMS_LAT");
>>>>DMS_LAT.setWidth(15);
>>>>DMS_LAT.setSqlType(0);
>>>>LONG.setCaption("LONG");
>>>>LONG.setColumnName("LONG");
>>>>LONG.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>LONG.setPreferredOrdinal(4);
>>>>LONG.setServerColumnName("LONG");
>>>>LONG.setWidth(15);
>>>>LONG.setSqlType(0);
>>>>LAT.setCaption("LAT");
>>>>LAT.setColumnName("LAT");
>>>>LAT.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>LAT.setPreferredOrdinal(3);
>>>>LAT.setServerColumnName("LAT");
>>>>LAT.setWidth(15);
>>>>LAT.setSqlType(0);
>>>>UNI.setCaption("UNI");
>>>>UNI.setColumnName("UNI");
>>>>UNI.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>UNI.setPreferredOrdinal(2);
>>>>UNI.setServerColumnName("UNI");
>>>>UNI.setWidth(15);
>>>>UNI.setSqlType(0);
>>>>UFI.setCaption("UFI");
>>>>UFI.setColumnName("UFI");
>>>>UFI.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>UFI.setPreferredOrdinal(1);
>>>>UFI.setServerColumnName("UFI");
>>>>UFI.setWidth(15);
>>>>UFI.setSqlType(0);
>>>>RC.setCaption("RC");
>>>>RC.setColumnName("RC");
>>>>RC.setDataType(com.borland.dx.dataset.Variant.STRING);
>>>>RC.setPreferredOrdinal(0);
>>>>RC.setWidth(1);
>>>>RC.setServerColumnName("RC");
>>>>RC.setSqlType(0);
>>>>
>>>>
>>>>tableDataSet1.setColumns(new Column[]
>>>>{RC, UFI, UNI, LAT, LONG, DMS_LAT,
>>>>DMS_LONG, UTM,
>>>>JOG, FC, DSG, PC, CC1, ADM1, ADM2, DIM,
>>>>CC2, NT,
>>>>LC, SHORT_FORM, GENERIC, SHORT_NAME,
>>>>FULL_NAME,
>>>>FULL_NAME_ND, MOD_DATE});
>>>>tableDataSet1.setTableName("coordinates");
>>>>tableDataSet1.setSchemaName("st_eis");
>>>>
>>>>textDataFile.setDelimiter("");
>>>>textDataFile.setEncoding("Cp1252");
>>>>
>>>>textDataFile.setFileFormat(com.borland.dx.dataset.DataFileFormat.ASCII);
>>>>
>>>>
>>>>textDataFile.setSeparator("\t");
>>>>textDataFile.setLoadAsInserted(true);
>>>>
>>>>try
>>>>{
>>>>
>>>>/**
>>>>==================================================================*/
>>>>/** Removing first line (headers) */
>>>>/**
>>>>==================================================================*/
>>>>for (int i = 0; i < fileNames.length; i++)
>>>>{
>>>>String removerPath = filePath + "\\" + fileNames[i] + ".txt";
>>>>RandomAccessFile file = new RandomAccessFile(removerPath,
>>>>"rws");
>>>>file.seek(0);
>>>>String line1 = file.readLine();
>>>>byte content[] = new byte[line1.length()];
>>>>file.seek(0);
>>>>file.write(content);
>>>>file.close();
>>>>}
>>>>
>>>>//
>>>>==================================================================
>>>>// Importing
>>>>//
>>>>==================================================================
>>>>outputArea.append("Importing...:");
>>>>database1.openConnection();
>>>>for (int i = 0; i < fileNames.length; i++)
>>>>{
>>>>
>>>>String absolutePath = filePath + "\\" + fileNames[i] + ".txt";
>>>>textDataFile.setFileName(absolutePath);
>>>>textDataFile.load(tableDataSet1);
>>>>System.out.println("Current file: " + absolutePath);
>>>>outputArea.append("\nRows loaded: " +
>>>>tableDataSet1.getRowCount());
>>>>database1.saveChanges(tableDataSet1);
>>>>tableDataSet1.clearValues();
>>>>
>>>>}
>>>>database1.closeConnection();
>>>>outputArea.append("\nImport completed!");
>>>>}
>>>>
>>>>catch (Exception ex)
>>>>{
>>>>ex.printStackTrace();
>>>>throw new Exception();
>>>>}
>>>>
>>>>}
>>>>
>>>>}
>>>>
>>>>
>>>><---Code Snippet------------------------------------------------------>
>>>>
>>>>Thanks,
>>>>Michael
>>>
>>>
>>>
--
=============================================
TeamB are volunteer helpers. Please DO NOT REPLY VIA EMAIL!
Post all questions and replies to this newsgroup ONLY
For papers on DataExpress, Applets, JSP, and Web Development go to:
www.microps.com/mps/paperFAQ.html
====================================================