#!/usr/bin/perl
use Unicode::String qw(utf8);
use XML::XPath;
use XML::LibXML;
use Tk;
use Tk::DropSite 'Win32';
use Tk::Text;

my $mw = MainWindow->new( -background => '#E0F0FF', );

$mw->Label(
  -text       => 'Script_RegExp : Faire un glisser-deposer du repertoire a traiter avec le script contenant des expressions regulieres',
  -background => '#E0F0FF',
)->pack(qw/ -side top -pady 2/);

my $widgetRegExp= $mw->Scrolled(
  'Text',
  -background => 'white',
  -scrollbars => 'osoe',
  -wrap       => 'none',
  -height     => 7,
)->pack(qw/ -side top -pady 2/);

$mw->Label(
  -text       => 'Script_LibXml : Faire un glisser-deposer du repertoire a traiter avec le script utilisant le module XML::LibXML',
  -background => '#E0F0FF',
)->pack(qw/ -pady 2/);

my $widgetLibXml = $mw->Scrolled(
  'Text',
  -background => 'white',
  -scrollbars => 'osoe',
  -wrap       => 'none',
  -height     => 7,
)->pack(qw/ -pady 2 /);

$mw->Label(
  -text       => 'Script_XmlXpath : Faire un glisser-deposer du repertoire a traiter avec le script utilisant le module XML::XPath',
  -background => '#E0F0FF',
)->pack(qw/ -pady 3/);

my $widgetXmlXpath = $mw->Scrolled(
  'Text',
  -background => 'white',
  -scrollbars => 'osoe',
  -wrap       => 'none',
  -height     => 7,
)->pack(qw/ -pady 3 /);


# Conception du glisser-déposer
$widgetRegExp->DropSite(
  -dropcommand => [ \&ActionDrop_regexp, $widgetRegExp],
  -droptypes   => 'Win32',
);
$widgetLibXml->DropSite(
  -dropcommand => [ \&ActionDrop_libxml, $widgetLibXml],
  -droptypes   => 'Win32',
);
$widgetXmlXpath->DropSite(
  -dropcommand => [ \&ActionDrop_xpath, $widgetXmlXpath],
  -droptypes   => 'Win32',
);
MainLoop;

sub ActionDrop_regexp {
  my ( $widgetRegExp ) = @_;

  my $sel = $widgetRegExp->SelectionGet( -selection => 'CLIPBOARD' );
  foreach my $fichier_rep ( split( /\n/, $sel ) ) {
    $widgetRegExp->insert( 'end', "$fichier_rep\n" );
	$widgetRegExp->SelectionOwn(-command=>&RegExp($fichier_rep));
  }
  return;
} 

sub ActionDrop_libxml {
  my ( $widgetLibXml ) = @_;

  my $sel = $widgetLibXml->SelectionGet( -selection => 'CLIPBOARD' );
  foreach my $fichier_rep ( split( /\n/, $sel ) ) {
    $widgetLibXml->insert( 'end', "$fichier_rep\n" );
	$widgetLibXml->SelectionOwn(-command=>&LibXml($fichier_rep));
  }
  return;
} 

sub ActionDrop_xpath {
  my ( $widgetXmlXpath ) = @_;
  
  my $sel = $widgetXmlXpath->SelectionGet( -selection => 'CLIPBOARD' );
  foreach my $fichier_rep ( split( /\n/, $sel ) ) {
    $widgetXmlXpath->insert( 'end', "$fichier_rep\n" );
	$widgetXmlXpath->SelectionOwn(-command=>&Xpath($fichier_rep));
  }
  return;
}


#################################################################
#---------- RegExp ---------------------------------------------
#################################################################
sub RegExp {
my $rep = $_[0] || '';
$rep=~ s/[\/]$//;

my $dossier = $rep;
$dossier =~ s/.+\\(.+)$/$1/;

mkdir("resultat_regxp_$dossier");
mkdir("./resultat_regxp_$dossier/resultat");
mkdir("./resultat_regxp_$dossier/cordial");
mkdir("./resultat_regxp_$dossier/cordial/fichiers");

my %tabcontenu = ();

#-----------------------------------------------------------------
&parcoursarborescencefichiers_RegExp($rep,$dossier);	#recurse!
#------------------------------------------------------------------

$cheminrep ="./resultat_regxp_$dossier/resultat";
opendir(DIRS,$cheminrep) or die "can't open $cheminrep: $!\n";
my @files = readdir(DIRS);
closedir(DIRS);
foreach my $file(@files){
	next if $file =~ /^\.\.?$/;
			if($file =~/\.xml$/){
				open(OUT,">>:encoding(utf-8)","./resultat_regxp_$dossier/resultat/$file");
				print OUT "</file>\n";
				close(OUT);
			}
		if($file=~/\.txt$/){
	open(IN,"<:encoding(utf8)","./resultat_regxp_$dossier/resultat/$file");
	open (OUT4, ">:encoding(iso-8859-1)","./resultat_regxp_$dossier/cordial/iso_$file");
	while(<IN>){
		 print OUT4 $_; 
	}
		close (OUT4);
		close (IN);
	}
}

my $methode = "regxp";
$cordial="./resultat_regxp_$dossier/cordial";
opendir(COR, $cordial) or die "can't open $path: $!\n";
my @fics = readdir(COR);
closedir(COR);
foreach my $fich(@fics){
	next if $fich =~ /^\.\.?$/;
	&decoupe($fich,$dossier,$methode); # Procédure qui permet un fichier en plusieurs fichiers de 2Mo
}

system("del texteaetiqueter.txt");
system("del treetagger.txt");
system("del treetagger.txt.xml");

$response = $mw->messageBox(-icon => 'info', -message => 'Extraction terminée !', -title => 'Extraction finie', -type => 'OK');

exit;
}

sub parcoursarborescencefichiers_RegExp {
    my $path = shift(@_);
	my $dossier = shift(@_);
	
    opendir(DIR, $path) or die "can't open $path: $!\n";
    my @files = readdir(DIR);
    closedir(DIR);

    foreach my $file (@files) {
		next if $file =~ /^\.\.?$/;
		$file = $path."/".$file;
		if (-d $file) {
			&parcoursarborescencefichiers_RegExp($file);	#recurse!
		}
		if (-f $file) {
			if(($file=~/\.xml$/) && ($file!~/\/fil.+\.xml$/) && ($file !~/0,2-3404,1-0,0\.xml$/)) {
			my $rubrique="";
			my $encodage = "";
			my $encodagesortie="utf-8";
			my $texte="";
			if (-z $file) {
			    print "$file est vide: pas de traitement\n";
			}else{
			        open(FIC,$file);
			        while (my $ligne=<FIC>) {
				    $ligne =~ s/\n//g;
				    if($ligne =~/(iso-8859-1|utf-8)/ig){
					    $encodage = $1;
				    }
			    }			
			    close(FIC);
			
			    open(FILE,"<:encoding($encodage)",$file);
			    while (my $ligne=<FILE>){
				    $ligne =~ s/\n//g;
				    $texte .= $ligne;
					}
				 close(FILE);
			        if ($texte=~/<channel><title>([^<]+)<\/title>/){
			            $rubrique = $1;  
			            $rubrique=~s/é/e/gi;
	    	            $rubrique=~s/è/e/gi;
		    	        $rubrique=~s/ê/e/gi;
		    	        $rubrique=~s/à/a/gi;
		                $rubrique=~s/es /e/i;
			            $rubrique=~ s/Le ?Monde.fr ?://;
	    	            $rubrique=~ s/ ?- ?Le ?Monde.fr//;
	    	            $rubrique=~ s/es$/e/i;
	   		            $rubrique=~ s/ //g;
	    	            $rubrique=uc($rubrique);
		    	    }
			    			   
				if($rubrique eq ""){
					$rubrique = "non-classe";
				}   
				
				open(OUT1,">>:encoding(utf-8)","./resultat_regxp_$dossier/resultat/$rubrique.txt");
		        open(OUT2,">>:encoding(utf-8)","./resultat_regxp_$dossier/resultat/$rubrique.xml");
				open(OUT3,">>:encoding(utf-8)","./resultat_regxp_$dossier/resultat/$rubrique-treetagger.xml");
			    if(-z OUT2){	
				    print OUT2 "<?xml version=\"1.0\" encoding=\"$encodagesortie\" ?>\n";
				    print OUT2 "<file>\n";
				    print OUT2 "<name>$ARGV[0]</name>\n";
			    }
				    if(-z OUT3){	
				    print OUT3 "<?xml version=\"1.0\" encoding=\"$encodagesortie\" ?>\n";
				    print OUT3 "<file>\n";
				    print OUT3 "<name>$ARGV[0]</name>\n";
			    }
		        $texte =~ s/> *</></g;
			    $texte=~/<pubDate>([^<]+)<\/pubDate>/;
			    my $date=$1;
			    if (uc($encodage) ne "UTF-8") {utf8($date);}
			    print OUT2 "<date>".$date."</date>\n";
			    print OUT2 "<items>\n";
			    print OUT3 "<date>".$date."</date>\n";
			    print OUT3 "<items>\n";
			    while ($texte =~ /<item><title>(.+?)<\/title>.+?<description>(.+?)<\/description>/g) {
				    my $titre=$1;
				    my $resume=$2;
				    my $test = $titre;
				    if(!exists $tabcontenu{$test}){
				        if (uc($encodage) ne "UTF-8") {utf8($titre);utf8($resume);}
					        $titre = &nettoietexte($titre);
					        $resume = &nettoietexte($resume);
							my ($titreetiquete, $resumeetiquete) = &etiquetage ($titre, $resume);
					        print OUT1 "Titre : $titre \n";
					        print OUT1 "Resume : $resume \n";
					        print OUT2 "<item><title>$titre</title><abstract>$resume</abstract></item>\n";
							print OUT3 "<item>\n<title>\n$titreetiquete</title>\n<abstract>\n$resumeetiquete</abstract>\n</item>\n";
					        $tabcontenu{$test}++;
				        }
				    }
				print OUT2 "</items>\n";
				print OUT3 "</items>\n";
				close(OUT1);
				close(OUT2);
				close(OUT3);
			    }
		    }
		}
	}
}

#################################################################
#---------- LibXml ---------------------------------------------
#################################################################
sub LibXml {
my $rep = $_[0] || '';
$rep=~ s/[\/]$//;

my $dossier = $rep;
$dossier =~ s/.+\\(.+)$/$1/;

mkdir("resultat_libxml_$dossier");
mkdir("./resultat_libxml_$dossier/resultat");
mkdir("./resultat_libxml_$dossier/cordial");
mkdir("./resultat_libxml_$dossier/cordial/fichiers");

my %tabcontenu = ();

#-----------------------------------------------------------------
&parcoursarborescencefichiers_LibXml($rep,$dossier);	#recurse!
#------------------------------------------------------------------

$cheminrep ="./resultat_libxml_$dossier/resultat";
opendir(DIRS,$cheminrep) or die "can't open $cheminrep: $!\n";
my @files = readdir(DIRS);
closedir(DIRS);
foreach my $file(@files){
	next if $file =~ /^\.\.?$/;
			if($file =~/\.xml$/){
				open(OUT,">>:encoding(utf-8)","./resultat_libxml_$dossier/resultat/$file");
				print OUT "</file>\n";
				close(OUT);
			}
		if($file=~/\.txt$/){
	open(IN,"<:encoding(utf8)","./resultat_libxml_$dossier/resultat/$file");
	open (OUT4, ">:encoding(iso-8859-1)","./resultat_libxml_$dossier/cordial/iso_$file");
	while(<IN>){
		 print OUT4 $_; 
	}
		close (OUT4);
		close (IN);
	}
}

my $methode = "libxml";
$cordial="./resultat_libxml_$dossier/cordial";
opendir(COR, $cordial) or die "can't open $path: $!\n";
my @fics = readdir(COR);
closedir(COR);
foreach my $fich(@fics){
	next if $fich =~ /^\.\.?$/;
	&decoupe($fich,$dossier,$methode); # Procédure qui permet un fichier en plusieurs fichiers de 2Mo
}

system("del texteaetiqueter.txt");
system("del treetagger.txt");
system("del treetagger.txt.xml");

$response = $mw->messageBox(-icon => 'info', -message => 'Extraction terminée !', -title => 'Extraction finie', -type => 'OK');

exit;
}

sub parcoursarborescencefichiers_LibXml {
    my $path = shift(@_);
	my $dossier = shift(@_);
    
	opendir(DIR, $path) or die "can't open $path: $!\n";
    my @files = readdir(DIR);
    closedir(DIR);
    foreach my $file (@files) {
		next if $file =~ /^\.\.?$/;
		$file = $path."/".$file;
		if (-d $file) {
			&parcoursarborescencefichiers_LibXml($file);	#recurse!
		}
		if (-f $file) {
			if(($file=~/\.xml$/) && ($file!~/\/fil.+\.xml$/) && ($file !~/0,2-3404,1-0,0\.xml$/)){
			my $encodage = "";
			my $encodagesortie="utf-8";
			my $texte="";
			if(-z $file){
				print "$file est vide pas de traitement\n";
			}else{
			    open(FIC,$file);
				while (my $ligne=<FIC>) {
					$ligne =~ s/\n//g;
					if($ligne =~/(iso-8859-1|utf-8)/ig){
						$encodage = $1;
					}
				}
				close(FIC);
				open(FILE,"<:encoding($encodage)", $file);
				my $texte="";
				while (my $ligne=<FILE>) {
					$ligne =~ s/\n//g;
					$ligne =~ s/\r//g;
					$texte .= $ligne;
				    
			    }
				close(FILE);
				
				if($texte =~/<channel><title>([^<]+)<\/title>/ig){
					$rubrique = $1;
					$rubrique=~s/é/e/gi;
					$rubrique=~s/è/e/gi;
					$rubrique=~s/ê/e/gi;
					$rubrique=~s/à/a/gi;
					$rubrique=~ s/Le ?Monde.fr ?://;
					$rubrique=~s/ ?- ?Le ?Monde\.fr//;
					$rubrique=~s/es$/e/i;
					$rubrique=~ s/ //g;
					$rubrique=uc($rubrique);
				}
								
				if($rubrique eq ""){
					$rubrique = "NONCLASSE";
				}
				open(OUT1,">>:encoding(utf-8)","./resultat_libxml_$dossier/resultat/$rubrique.txt");
				open(OUT2,">>:encoding(utf-8)","./resultat_libxml_$dossier/resultat/$rubrique.xml");
				open(OUT3,">>:encoding(utf-8)","./resultat_libxml_$dossier/resultat/$rubrique-treetagger.xml");
				if(-z OUT2){
					print OUT2 "<?xml version=\"1.0\" encoding=\"$encodagesortie\" ?>\n";
					print OUT2 "<file>\n";
					print OUT2 "<name>$ARGV[0]</name>\n";
				}
				    if(-z OUT3){	
				    print OUT3 "<?xml version=\"1.0\" encoding=\"$encodagesortie\" ?>\n";
				    print OUT3 "<file>\n";
				    print OUT3 "<name>$ARGV[0]</name>\n";
			    }
				print OUT2 "<items>\n";
				print OUT3 "<items>\n";
			
				my $input_file= $file;
				my $parser = XML::LibXML->new();
				my $xp = $parser->parse_file($input_file);
				my $date = "";
				foreach my $noeud_date ( $xp->findnodes('//channel')->get_nodelist ) {
					$date=$noeud_date->findnodes('//pubDate')->string_value;
					if (uc($encodage) ne "UTF-8") {utf8($date);}
				}
				print OUT2 "<date>$date</date>\n";
				print OUT3 "<date>\n$date</date>\n";
				
				foreach my $noeud ( $xp->findnodes('//item')->get_nodelist ) {
					my $titre=$noeud->findnodes('title')->string_value;
					my $resume=$noeud->findnodes('description')->string_value;				
					my $test=$titre;
					if(!exists $tabcontenu{$test}){
						if (uc($encodage) ne "UTF-8") {utf8($titre);utf8($resume);}
						$titre=&nettoietexte($titre);
					    $resume=&nettoietexte($resume);
						my ($titreetiquete, $resumeetiquete) = &etiquetage ($titre, $resume);

						print OUT1 "Titre : $titre \n";
					    print OUT1 "Resume : $resume \n";
					    print OUT2 "<item><title>$titre</title><abstract>$resume</abstract></item>\n";
						print OUT3 "<item>\n<title>\n$titreetiquete</title>\n<abstract>\n$resumeetiquete</abstract>\n</item>\n";

						$tabcontenu{$test}++;
					}
				}
				print OUT2 "</items>\n";
				print OUT3 "</items>\n";
				close(OUT1);
				close(OUT2);
				close(OUT3)
			}
		}
		}
	}
}


#################################################################
#---------- Xpath ---------------------------------------------
#################################################################
sub Xpath {
my $rep = $_[0] || '';
$rep=~ s/[\/]$//;

my $dossier = $rep;
$dossier =~ s/.+\\(.+)$/$1/;

mkdir("resultat_xpath_$dossier");
mkdir("./resultat_xpath_$dossier/resultat");
mkdir("./resultat_xpath_$dossier/cordial");
mkdir("./resultat_xpath_$dossier/cordial/fichiers");

my %tabcontenu = ();

#-----------------------------------------------------------------
&parcoursarborescencefichiers_Xpath($rep,$dossier);	#recurse!
#------------------------------------------------------------------

$cheminrep ="./resultat_xpath_$dossier/resultat";
opendir(DIRS,$cheminrep) or die "can't open $cheminrep: $!\n";
my @files = readdir(DIRS);
closedir(DIRS);
foreach my $file(@files){
	next if $file =~ /^\.\.?$/;
			if($file =~/\.xml$/){
				open(OUT,">>:encoding(utf-8)","./resultat_xpath_$dossier/resultat/$file");
				print OUT "</file>\n";
				close(OUT);
			}
		if($file=~/\.txt$/){
	open(IN,"<:encoding(utf8)","./resultat_xpath_$dossier/resultat/$file");
	open (OUT4, ">:encoding(iso-8859-1)","./resultat_xpath_$dossier/cordial/iso_$file");
	while(<IN>){
		 print OUT4 $_; 
	}
		close (OUT4);
		close (IN);
	}
}

my $methode = "xpath";
$cordial="./resultat_xpath_$dossier/cordial";
opendir(COR, $cordial) or die "can't open $path: $!\n";
my @fics = readdir(COR);
closedir(COR);
foreach my $fich(@fics){
	next if $fich =~ /^\.\.?$/;
	&decoupe($fich,$dossier,$methode); # Procédure qui permet un fichier en plusieurs fichiers de 2Mo
}

system("del texteaetiqueter.txt");
system("del treetagger.txt");
system("del treetagger.txt.xml");

$response = $mw->messageBox(-icon => 'info', -message => 'Extraction terminée !', -title => 'Extraction finie', -type => 'OK');

exit;
}

sub parcoursarborescencefichiers_Xpath {
    my $path = shift(@_);
    my $dossier = shift(@_);
	
	opendir(DIR, $path) or die "can't open $path: $!\n";
    my @files = readdir(DIR);
    closedir(DIR);
    foreach my $file (@files) {
		next if $file =~ /^\.\.?$/;
		$file = $path."/".$file;
		if (-d $file) {
			&parcoursarborescencefichiers_Xpath($file);	#recurse!
		}
		if (-f $file) {
			if(($file=~/\.xml$/) && ($file!~/\/fil.+\.xml$/) && ($file !~/0,2-3404,1-0,0\.xml$/)){
			my $encodage = "";
			my $encodagesortie="utf-8";
			my $texte="";
			
			if(-z $file){
				print "$file est vide pas de traitement\n";
			}else{
			    open(FIC,$file);
				while (my $ligne=<FIC>) {
					$ligne =~ s/\n//g;
					if($ligne =~/(iso-8859-1|utf-8)/ig){
						$encodage = $1;
					}
				}
				close(FIC);
				#-----------------------------------------------------------
				open(FILE,"<:encoding($encodage)", $file);
				my $texte="";
				while (my $ligne=<FILE>) {
					$ligne =~ s/\n//g;
					$ligne =~ s/\r//g;
					 $texte .= $ligne;
				    
			    }
				
					if($texte =~/<channel><title>([^<]+)<\/title>/ig){
						$rubrique = $1;
						$rubrique=~s/é/e/gi;
						$rubrique=~s/è/e/gi;
						$rubrique=~s/ê/e/gi;
						$rubrique=~s/à/a/gi;
						$rubrique=~ s/Le ?Monde.fr ?://;
						$rubrique=~s/ ?- ?Le ?Monde\.fr//;
						$rubrique=~s/es$/e/i;
						$rubrique=~ s/ //g;
						$rubrique=uc($rubrique);
					}
				
				close(FILE);
				if($rubrique eq ""){
					$rubrique = "NONCLASSE";
				}
				
				#-----------------------------------------------------------
				open(OUT1,">>:encoding(utf-8)","./resultat_xpath_$dossier/resultat/$rubrique.txt");
				open(OUT2,">>:encoding(utf-8)","./resultat_xpath_$dossier/resultat/$rubrique.xml");
				open(OUT3,">>:encoding(utf-8)","./resultat_xpath_$dossier/resultat/$rubrique-treetagger.xml");
				if(-z OUT2){
					print OUT2 "<?xml version=\"1.0\" encoding=\"$encodagesortie\" ?>\n";
					print OUT2 "<file>\n";
					print OUT2 "<name>$ARGV[0]</name>\n";
				}
				    if(-z OUT3){	
				    print OUT3 "<?xml version=\"1.0\" encoding=\"$encodagesortie\" ?>\n";
				    print OUT3 "<file>\n";
				    print OUT3 "<name>$ARGV[0]</name>\n";
			    }
				#-----------------------------------------------------------
				print OUT2 "<items>\n";
				print OUT3 "<items>\n";
				my $input_file= $file;
				my $xp = XML::XPath->new( filename => $input_file ) or die "big trouble";
				my $date = "";
				my $date_path="//channel";
				foreach my $date_noeud ( $xp->find($date_path)->get_nodelist ) {
					$date=$date_noeud->find('pubDate')->string_value;
				}
				print OUT2 "<date>$date</date>\n";
				print OUT3 "<date>\n$date</date>\n";
				my $search_path.="//item";
				foreach my $noeud ( $xp->find($search_path)->get_nodelist ) {
					my $titre=$noeud->find('title')->string_value;
					my $resume=$noeud->find('description')->string_value;
	                $titre=&nettoietexte($titre);
	                $resume=&nettoietexte($resume);
					my $test=$titre;
					if (uc($encodage) ne "UTF-8") {utf8($titre);utf8($resume);}
					if(!exists $tabcontenu{$test}){
							my ($titreetiquete, $resumeetiquete) = &etiquetage ($titre, $resume);
					        print OUT1 "Titre : $titre \n";
					        print OUT1 "Resume : $resume \n";
					        print OUT2 "<item><title>$titre</title><abstract>$resume</abstract></item>\n";
							print OUT3 "<item>\n<title>\n$titreetiquete</title>\n<abstract>\n$resumeetiquete</abstract>\n</item>\n";
					        $tabcontenu{$test}++;
					}
				}
				print OUT2 "</items>\n";
				print OUT3 "</items>\n";
				close(OUT1);
				close(OUT2);
				close(OUT3);
			}
		}
		}
	}
}


#################################################################
#---------- Tree-Tagger -----------------------------------------
#################################################################
sub etiquetage($titre, $resume) {
    my ($titre, $resume) = @_;
	my $tmptag="texteaetiqueter.txt";
	open (TMPFILE,">:encoding(utf-8)", $tmptag);
	print TMPFILE $titre,"\n";
	close(TMPFILE);
	my $tokenisefr = "\"./tree-tagger/tokenise-fr\"";
	my $treetaggerexe = "\"./tree-tagger/tree-tagger\"";
	my $treetagger2xml = "\"./tree-tagger/treetagger2xml\"";
	system("$tokenisefr $tmptag | $treetaggerexe ./tree-tagger/french-utf8.par -lemma -token -no-unknown -sgml > treetagger.txt");
	system("$treetagger2xml treetagger.txt utf-8");
	# lecture du resultat tagge en xml :
	open(OUT,"<:encoding(utf-8)","treetagger.txt.xml");
    my $firstline=<OUT>;
	my $titreetiquete="";
	while (my $l=<OUT>) {
	   $titreetiquete.=$l;
	}
	close(OUT);
	#----- le resume
	open (TMPFILE,">:encoding(utf-8)", $tmptag);
	print TMPFILE $resume,"\n";
	close(TMPFILE);
	system("$tokenisefr $tmptag | $treetaggerexe ./tree-tagger/french-utf8.par -lemma -token -no-unknown -sgml > treetagger.txt");
	system("$treetagger2xml treetagger.txt utf-8");
	# lecture du resultat tagge en xml :
	open(OUT,"<:encoding(utf-8)","treetagger.txt.xml");
    my $firstline=<OUT>;
	my $resumeetiquete="";
	while (my $l=<OUT>) {
	   $resumeetiquete.=$l;
	}
	close(OUT);
    # on renvoie les resultats :
	return ($titreetiquete,$resumeetiquete);
}

#################################################################
#---------- NettoieTexte ---------------------------------------
#################################################################
sub nettoietexte {
    my $texte=shift;
    $texte =~ s/&lt;/</g;
    $texte =~ s/&gt;/>/g;
    $texte =~ s/<a href[^>]+>//g;
    $texte =~ s/<img[^>]+>//g;
    $texte =~ s/<\/a>//g;
    $texte =~ s/&#38;#39;/'/g;
    $texte =~ s/&#38;#34;/"/g;
    $texte =~ s/&#233;/é/g;
    $texte =~ s/&#234;/ê/g;
    $texte =~ s/<[^>]+>//g;
    $texte =~ s/&nbsp;/ /g;
    $texte=~s/&#39;/'/g;
    $texte=~s/&#34;/"/g;
    $texte=~s/&amp;#39;/'/g;
    $texte=~s/&amp;#34;/"/g;
    return $texte;
}

#################################################################
#---------- Decoupe pour Cordial --------------------------------
#################################################################
sub decoupe{
my $fichier = shift(@_);
my $dossier = shift(@_);
my $methode = shift(@_);
my $chemin = "./resultat_$methode";
$chemin .= "_$dossier/cordial";
open(F,"$chemin/$fichier");
my $num = 1;
my $taille = 0;
while($ligne = <F>) { 
	if($taille < 2000000){ # On veut des fichiers de 2Mo = 2000000 octets
	open (OUT,">>$chemin/fichiers/$fichier".$num.".txt");
	print OUT $ligne;
	$taille = -s OUT;
	close(OUT);
	}else{
     $num++;
	 $taille = 0;
	}
}
close (F);
} 


