Flat Previous Topic | Next Topic
Yutaka
Posted on: 1/4/2010 9:51 pm
Webmaster
Joined: 9/28/2006
From: Redmond
Posts: 2423
Re: how to delete duplicate lines

``````
function Pair( i, s )
{
this.index = i;
this.str = s;
}

nLines = document.GetLines();

// Create an array
a = new Array( nLines );

// Fill the array a with all lines (with returns) in the document.
for( i = 1; i <= nLines; i++ ) {
if( (i % 1000) == 0 ){
status = "Reading lines: " + String(i + 1) + "/" + String(nLines);
}
var pair = new Pair( i, document.GetLine( i, eeGetLineWithNewLines ) );
a.push( pair );
}

status = "Sorting lines..."

a.sort( function(a,b){
if( a.str > b.str ){
return 1;
}
if( a.str < b.str ){
return -1;
}
return a.index - b.index;
});

// Delete duplicate elements.
for( i = 1; i < nLines; i++ ){
if( (i % 10) == 0 ){
status = "Deleting duplicate lines: " + String(i + 1) + "/" + String(nLines);
}
if( a[i].str == a[i-1].str ){
a[i].index = 0;  // disable
}
}

status = "Sorting lines again..."

a.sort( function(a,b){
return a.index - b.index;
});

var str = "";
n = 0;
for( i = 0; i < nLines; i++ ){
if( a[i].index != 0 ){
if( (i % 1000) == 0 ){
status = "Joining lines: " + String(i + 1) + "/" + String(nLines);
}
str += a[i].str;
}
else {
n++;
}
}

// Replace the entire document with new elements
document.selection.SelectAll();
document.selection.Text = str;
status = n + " duplicate lines deleteded."
``````

----------------
Yutaka Emura
Developer of EmEditor
http://www.emeditor.com/

 Flat Previous Topic | Next Topic

Subject Poster Date
how to delete duplicate lines user 9/29/2006 7:27 am
Re: how to delete duplicate lines Yutaka 9/29/2006 11:56 am
Re: how to delete duplicate lines user 9/29/2006 12:42 pm
Re: how to delete duplicate lines Hellados 4/18/2009 1:10 am
Re: how to delete duplicate lines Yutaka 4/18/2009 1:26 pm
Re: how to delete duplicate lines Salabim 1/3/2010 10:33 am
» Re: how to delete duplicate lines Yutaka 1/4/2010 9:51 pm
Re: how to delete duplicate lines Salabim 1/5/2010 11:07 pm
Re: how to delete duplicate lines Deipotent 8/10/2011 12:21 pm
Re: how to delete duplicate lines Yutaka 8/10/2011 1:11 pm
Re: how to delete duplicate lines Deipotent 8/10/2011 3:50 pm
Re: how to delete duplicate lines Yutaka 8/10/2011 4:37 pm
Re: how to delete duplicate lines Monkeyman 6/5/2011 12:47 pm
Re: how to delete duplicate lines raikrivera 7/28/2011 1:36 am

 Register To Post ________________Announcements  - Announcements ________________EmEditor Core  - Questions and Answers about EmEditor Core   - Regular Expressions   - EmEditor Core Enhancement Suggestions   - EmEditor Core Bug Reports ________________Plug-ins  - Questions and Answers about Plug-ins   - Enhancement Suggestions about Plug-ins   - Plug-in Bug Reports ________________Macros  - Questions and Answers about Macros   - Enhancement Suggestions about Macros   - Macro Samples ________________Beta Room  - Beta Version Bug Reports ________________User Comments  - Testimonials ________________Other Topics  - Other Programs   - General