ascend4/TK/mergedat.tcl

#  mergedat.tcl: merge data file utilities for mashing integrator output
#  by Benjamin Allan
#  Created March 29, 1998
#  Part of ASCEND
#  Revision: $Revision: 1.6 $
#  Last modified on: $Date: 1998/06/18 15:55:30 $
#  Last modified by: $Author: mthomas $
#  Revision control file: $RCSfile: mergedat.tcl,v $
#
#  This file is part of the ASCEND Tcl/Tk Interface.
#
#  Copyright (C) 1998 Carnegie Mellon University
#
#  The ASCEND Tcl/Tk Interface is free software; you can redistribute
#  it and/or modify it under the terms of the GNU General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  The ASCEND Tcl/Tk Interface is distributed in hope that it will be
#  useful, but WITHOUT ANY WARRANTY; without even the implied warranty
#  of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with the program; if not, write to the Free Software
#  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139 USA.  Check the
#  file named COPYING.  COPYING is found in ../compiler.

# merge data file utility functions for mashing
# integrator output .dat files into a single file
# or into a matlab format file.
#
# Requires the ascend extension stringcompact

# Usage:
# asc_merge_data_files <format> <output file name> <infile1> [infile2...]
# Merges the infiles given (as many as you like) into a new outputfile
# formatted for matlab, excel, or ascend.
#
# If the output file named already exists, it is overwritten.
#
# format is one of "ascend", "excel", "matlab".
#
# For ascend format,
# any time a set of headers matches the previous set of headers,
# it is eliminated. A new header will start a new data set.
#
# For excel format,
# duplicate headers are eliminated, and ascend headers are converted
# to a format that spreadsheet users find more congenial for import.
#
# For matlab format,
# duplicate headers are eliminated, but if a new set of headers
# is encountered, the conversion stops. outfile name should be
# something ending in .m.
#
# With the exception of headers, input is read/written line at a time,
# so if an error is encountered, the output may be partial.


proc mdf_check_output {file} {
  # verifies that output is writable, or errs, and defines
  # the matlab data matrix name.
  global ascmdfVect
  if {[file isdirectory $file] || \
      ([file exists $file] && ![file writable $file]) } {
    error "asc_merge_data_files called with bad output $file"
  }
  set ascmdfVect(outname) $file
  set ascmdfVect(matname) [file rootname [file tail $file]]
}

proc mdf_check_inputs {flist} {
  # verifies that input is readable text, or errs
  foreach file $flist {
    if {![file readable $file] || ![file isfile $file]} {
      error "asc_merge_data_files called with bad input $file"
    } 
  }
}

proc asc_merge_data_files {format outfile args} {
  # merges the files named in args into outfile, formatted according
  # to $format. currently 'ascend' and 'matlab' are understood.
  # For ascend format,
  # any time a set of headers matches the previous set of headers,
  # it is eliminated. A new header will start a new data set.
  # For matlab format,
  # duplicate headers are eliminated, but if a new set of headers
  # is encountered, the conversion stops. outfile name should be
  # something .m.
  #
  # With the exception of headers, input is handled line at a time,
  # so if an error is encountered, the output may be partial.
  global ascmdfVect
  if {[llength $args]==0} {
    error "asc_merge_data_files called without input files"
  }
  catch {unset ascmdfVect}
  mdf_check_output $outfile
  mdf_check_inputs $args
  # preprocessing
  set format [string tolower $format]
  switch $format {
  ascend -
  excel { # do nothing
    }
  matlab {
      set ascmdfVect(matlabheaderdone) 0
    }
  default {
      error \
        "asc_merge_data_files format output_file input_file \[input_file...\] where format is 'ascend','excel', or 'matlab' and input files are ascend .dat output files."
    }
  }
  # process input files
  set ofid [open $outfile "w+"]
  foreach file $args {
    set ifid [open $file "r+"]
    switch $format {
    ascend {
        set error [catch {mdf_ascend2ascend $ifid $ofid} errmsg]
      }
    excel {
        set error [catch {mdf_ascend2excel $ifid $ofid} errmsg]
      }
    matlab {
        set error [catch {mdf_ascend2matlab $ifid $ofid} errmsg]
      }
    }
    close $ifid
    if {$error} {break}
  }
  if {!$error} {
    # post processing
    switch $format {
    ascend -
    excel {
        # do nothing
      }
    matlab {
        puts $ofid "\];"
        puts $ofid "echo on"
        set column 1
        foreach i $ascmdfVect(oldvars) {
          set aname [lindex $i 1]
          # Handle the multisubscript arrays, converting each join to a .
          # the combinations here manage mixed integer/symbol subscripts
          # order here matters, else get redundant _ in output
          regsub -all -- {'\]\['} $aname . m1
          regsub -all -- {\]\['} $m1 . m2
          regsub -all -- {'\]\[} $m2 . m3
          regsub -all -- {\['} $m3 . m4
          regsub -all -- {\[} $m4 . m5
          regsub -all -- {'\]\.} $m5 . m6
          regsub -all -- {\]\.} $m6 . m7
          regsub -all -- {'\]} $m7 {} m8
          regsub -all -- {\]} $m8 {} m9
          regsub -all -- {\.} $m9 _ m10
          puts $ofid "$m10 = $ascmdfVect(matname)var(:,$column);"
          incr column
        }
        puts $ofid "echo off"
      }
    }
  }
  close $ofid
  if {$error} {
    error $errmsg
  }
}

proc mdf_ascend2ascend {infid outfid} {
  # internal function to strip extra headers from ascend dat files
  global ascmdfVect
  set state error
  if {![info exists ascmdfVect(oldvars)]} {
    set ascmdfVect(oldvars) "{ERROR reading input} {}"
  }
  set ok 1
  while {$ok} {
    set cc [gets $infid data]
    if {$cc<0} {break}
    set line [stringcompact $data]
    if {[string length $line] <= 0} {
      continue;
    }
    set c [lindex $line 0]
    switch -exact -- $c {
    DATASET {
        set ascmdfVect(newline1) $data; # keep in case next set is new 
        set state header
      }
    Observations: -
    States: {
        set ascmdfVect(newline2) $data
        set ascmdfVect(newvars) {}
      }
    default {
        switch -- [string index $line 0] {
        \{ {
            lappend ascmdfVect(newvars) $data
          }
        default {
            switch $state {
            error {
                error "DATASET not seen as expected in digestion"
              }
            header {
                set state coltitle
                set ascmdfVect(newtitle) $data
              }
            coltitle {
                set state rowminus
                set ascmdfVect(newminus) $data
              }
            rowminus {
                set state more
                if {[string compare \
                      $ascmdfVect(newvars) $ascmdfVect(oldvars)] != 0} {
                  set ascmdfVect(oldvars) $ascmdfVect(newvars)
                  puts $outfid $ascmdfVect(newline1)
                  puts $outfid $ascmdfVect(newline2)
                  foreach vdef $ascmdfVect(newvars) {
                    puts $outfid $vdef
                  }
                  puts $outfid $ascmdfVect(newtitle)
                  puts $outfid $ascmdfVect(newminus)
                }
              }
            more {
                puts $outfid $data
              }
            default {
                error "Unexpected line:\n$data\n in input file."
              }
            }
          }
        }
      }
    }
  }
}

proc mdf_ascend2excel {infid outfid} {
  # internal function to strip extra headers from ascend dat files
  # turns the var definitions on their sides as column titles in a row
  # after putting out the row of units. drops the dataset headers.
  global ascmdfVect
  set state error
  if {![info exists ascmdfVect(oldvars)]} {
    set ascmdfVect(oldvars) "{ERROR reading input} {}"
  }
  set ok 1
  while {$ok} {
    set cc [gets $infid data]
    if {$cc<0} {break}
    set line [stringcompact $data]
    if {[string length $line] <= 0} {
      continue;
    }
    set c [lindex $line 0]
    switch -exact -- $c {
    DATASET {
        set ascmdfVect(newline1) $data; # keep in case next set is new 
        set state header
      }
    Observations: -
    States: {
        set ascmdfVect(newline2) $data
        set ascmdfVect(newvars) {}
      }
    default {
        switch -- [string index $line 0] {
        \{ {
            lappend ascmdfVect(newvars) $data
          }
        default {
            switch $state {
            error {
                error "DATASET not seen as expected in digestion"
              }
            header {
                set state coltitle
                set ascmdfVect(newtitle) $data
              }
            coltitle {
                set state rowminus
                set ascmdfVect(newminus) $data
              }
            rowminus {
                set state more
                if {[string compare \
                      $ascmdfVect(newvars) $ascmdfVect(oldvars)] != 0} {
                  set ascmdfVect(oldvars) $ascmdfVect(newvars)
                  # puts $outfid $ascmdfVect(newline1)
                  # puts $outfid $ascmdfVect(newline2)
                  # write a line of units
                  foreach vdef $ascmdfVect(newvars) {
                    puts -nonewline $outfid "\t[lindex $vdef 2]"
                  }
                  puts $outfid ""
                  # write a line of names
                  foreach vdef $ascmdfVect(newvars) {
                    puts -nonewline $outfid "\t[lindex $vdef 1]"
                  }
                  puts $outfid ""
                  # puts $outfid $ascmdfVect(newtitle)
                  # puts $outfid $ascmdfVect(newminus)
                }
              }
            more {
                puts $outfid $data
              }
            default {
                error "Unexpected line:\n$data\n in input file."
              }
            }
          }
        }
      }
    }
  }
}

proc mdf_ascend2matlab {infid outfid} {
  # internal function to strip extra headers from ascend dat files
  # and generate a single matlab matrix
  global ascmdfVect
  set state error
  if {![info exists ascmdfVect(oldvars)]} {
    set ascmdfVect(oldvars) "{ERROR reading input} {}"
  }
  set ok 1
  while {$ok} {
    set cc [gets $infid data]
    if {$cc<0} {break}
    set line [stringcompact $data]
    if {[string length $line] <= 0} {
      continue;
    }
    set c [lindex $line 0]
    switch -exact -- $c {
    DATASET {
        set ascmdfVect(newline1) $data; # keep in case next set is new 
        set state header
      }
    Observations: -
    States: {
        set ascmdfVect(newline2) $data
        set ascmdfVect(newvars) {}
      }
    default {
        switch -- [string index $line 0] {
        \{ {
            lappend ascmdfVect(newvars) $data
          }
        default {
            switch $state {
            error {
                error "DATASET not seen as expected in digestion"
              }
            header {
                set state coltitle
                set ascmdfVect(newtitle) $data
              }
            coltitle {
                set state rowminus
                set ascmdfVect(newminus) $data
              }
            rowminus {
                set state more
                if {[string compare \
                      $ascmdfVect(newvars) $ascmdfVect(oldvars)] != 0} {
                  if {$ascmdfVect(matlabheaderdone)} {
                    error "Cannot merge unlike data sets into a matlab file"
                  }
                  set ascmdfVect(oldvars) $ascmdfVect(newvars)
                  puts $outfid "%%$ascmdfVect(newline1)"
                  puts $outfid "%%$ascmdfVect(newline2)"
                  foreach vdef $ascmdfVect(newvars) {
                    puts $outfid "%%$vdef"
                  }
                  puts $outfid "%%$ascmdfVect(newtitle)"
                  puts $outfid "%%$ascmdfVect(newminus)"
                  puts $outfid "echo off"
                  puts $outfid $ascmdfVect(matname)var=\[
                  set ascmdfVect(matlabheaderdone) 1
                } else {
                  puts stderr "DUPLICATE HEADER"
                }
              }
            more {
                puts $outfid $data
              }
            default {
                error "Unexpected line:\n$data\n in input file."
              }
            }
          }
        }
      }
    }
  }
}
1	# mergedat.tcl: merge data file utilities for mashing integrator output
2	# by Benjamin Allan
3	# Created March 29, 1998
4	# Part of ASCEND
5	# Revision: $Revision: 1.6 $
6	# Last modified on: $Date: 1998/06/18 15:55:30 $
7	# Last modified by: $Author: mthomas $
8	# Revision control file: $RCSfile: mergedat.tcl,v $
9	#
10	# This file is part of the ASCEND Tcl/Tk Interface.
11	#
12	# Copyright (C) 1998 Carnegie Mellon University
13	#
14	# The ASCEND Tcl/Tk Interface is free software; you can redistribute
15	# it and/or modify it under the terms of the GNU General Public
16	# License as published by the Free Software Foundation; either
17	# version 2 of the License, or (at your option) any later version.
18	#
19	# The ASCEND Tcl/Tk Interface is distributed in hope that it will be
20	# useful, but WITHOUT ANY WARRANTY; without even the implied warranty
21	# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22	# GNU General Public License for more details.
23	#
24	# You should have received a copy of the GNU General Public License
25	# along with the program; if not, write to the Free Software
26	# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139 USA. Check the
27	# file named COPYING. COPYING is found in ../compiler.
28
29	# merge data file utility functions for mashing
30	# integrator output .dat files into a single file
31	# or into a matlab format file.
32	#
33	# Requires the ascend extension stringcompact
34
35	# Usage:
36	# asc_merge_data_files <format> <output file name> <infile1> [infile2...]
37	# Merges the infiles given (as many as you like) into a new outputfile
38	# formatted for matlab, excel, or ascend.
39	#
40	# If the output file named already exists, it is overwritten.
41	#
42	# format is one of "ascend", "excel", "matlab".
43	#
44	# For ascend format,
45	# any time a set of headers matches the previous set of headers,
46	# it is eliminated. A new header will start a new data set.
47	#
48	# For excel format,
49	# duplicate headers are eliminated, and ascend headers are converted
50	# to a format that spreadsheet users find more congenial for import.
51	#
52	# For matlab format,
53	# duplicate headers are eliminated, but if a new set of headers
54	# is encountered, the conversion stops. outfile name should be
55	# something ending in .m.
56	#
57	# With the exception of headers, input is read/written line at a time,
58	# so if an error is encountered, the output may be partial.
59
60
61	proc mdf_check_output {file} {
62	# verifies that output is writable, or errs, and defines
63	# the matlab data matrix name.
64	global ascmdfVect
65	if {[file isdirectory $file] \|\| \
66	([file exists $file] && ![file writable $file]) } {
67	error "asc_merge_data_files called with bad output $file"
68	}
69	set ascmdfVect(outname) $file
70	set ascmdfVect(matname) [file rootname [file tail $file]]
71	}
72
73	proc mdf_check_inputs {flist} {
74	# verifies that input is readable text, or errs
75	foreach file $flist {
76	if {![file readable $file] \|\| ![file isfile $file]} {
77	error "asc_merge_data_files called with bad input $file"
78	}
79	}
80	}
81
82	proc asc_merge_data_files {format outfile args} {
83	# merges the files named in args into outfile, formatted according
84	# to $format. currently 'ascend' and 'matlab' are understood.
85	# For ascend format,
86	# any time a set of headers matches the previous set of headers,
87	# it is eliminated. A new header will start a new data set.
88	# For matlab format,
89	# duplicate headers are eliminated, but if a new set of headers
90	# is encountered, the conversion stops. outfile name should be
91	# something .m.
92	#
93	# With the exception of headers, input is handled line at a time,
94	# so if an error is encountered, the output may be partial.
95	global ascmdfVect
96	if {[llength $args]==0} {
97	error "asc_merge_data_files called without input files"
98	}
99	catch {unset ascmdfVect}
100	mdf_check_output $outfile
101	mdf_check_inputs $args
102	# preprocessing
103	set format [string tolower $format]
104	switch $format {
105	ascend -
106	excel { # do nothing
107	}
108	matlab {
109	set ascmdfVect(matlabheaderdone) 0
110	}
111	default {
112	error \
113	"asc_merge_data_files format output_file input_file \[input_file...\] where format is 'ascend','excel', or 'matlab' and input files are ascend .dat output files."
114	}
115	}
116	# process input files
117	set ofid [open $outfile "w+"]
118	foreach file $args {
119	set ifid [open $file "r+"]
120	switch $format {
121	ascend {
122	set error [catch {mdf_ascend2ascend $ifid $ofid} errmsg]
123	}
124	excel {
125	set error [catch {mdf_ascend2excel $ifid $ofid} errmsg]
126	}
127	matlab {
128	set error [catch {mdf_ascend2matlab $ifid $ofid} errmsg]
129	}
130	}
131	close $ifid
132	if {$error} {break}
133	}
134	if {!$error} {
135	# post processing
136	switch $format {
137	ascend -
138	excel {
139	# do nothing
140	}
141	matlab {
142	puts $ofid "\];"
143	puts $ofid "echo on"
144	set column 1
145	foreach i $ascmdfVect(oldvars) {
146	set aname [lindex $i 1]
147	# Handle the multisubscript arrays, converting each join to a .
148	# the combinations here manage mixed integer/symbol subscripts
149	# order here matters, else get redundant _ in output
150	regsub -all -- {'\]\['} $aname . m1
151	regsub -all -- {\]\['} $m1 . m2
152	regsub -all -- {'\]\[} $m2 . m3
153	regsub -all -- {\['} $m3 . m4
154	regsub -all -- {\[} $m4 . m5
155	regsub -all -- {'\]\.} $m5 . m6
156	regsub -all -- {\]\.} $m6 . m7
157	regsub -all -- {'\]} $m7 {} m8
158	regsub -all -- {\]} $m8 {} m9
159	regsub -all -- {\.} $m9 _ m10
160	puts $ofid "$m10 = $ascmdfVect(matname)var(:,$column);"
161	incr column
162	}
163	puts $ofid "echo off"
164	}
165	}
166	}
167	close $ofid
168	if {$error} {
169	error $errmsg
170	}
171	}
172
173	proc mdf_ascend2ascend {infid outfid} {
174	# internal function to strip extra headers from ascend dat files
175	global ascmdfVect
176	set state error
177	if {![info exists ascmdfVect(oldvars)]} {
178	set ascmdfVect(oldvars) "{ERROR reading input} {}"
179	}
180	set ok 1
181	while {$ok} {
182	set cc [gets $infid data]
183	if {$cc<0} {break}
184	set line [stringcompact $data]
185	if {[string length $line] <= 0} {
186	continue;
187	}
188	set c [lindex $line 0]
189	switch -exact -- $c {
190	DATASET {
191	set ascmdfVect(newline1) $data; # keep in case next set is new
192	set state header
193	}
194	Observations: -
195	States: {
196	set ascmdfVect(newline2) $data
197	set ascmdfVect(newvars) {}
198	}
199	default {
200	switch -- [string index $line 0] {
201	\{ {
202	lappend ascmdfVect(newvars) $data
203	}
204	default {
205	switch $state {
206	error {
207	error "DATASET not seen as expected in digestion"
208	}
209	header {
210	set state coltitle
211	set ascmdfVect(newtitle) $data
212	}
213	coltitle {
214	set state rowminus
215	set ascmdfVect(newminus) $data
216	}
217	rowminus {
218	set state more
219	if {[string compare \
220	$ascmdfVect(newvars) $ascmdfVect(oldvars)] != 0} {
221	set ascmdfVect(oldvars) $ascmdfVect(newvars)
222	puts $outfid $ascmdfVect(newline1)
223	puts $outfid $ascmdfVect(newline2)
224	foreach vdef $ascmdfVect(newvars) {
225	puts $outfid $vdef
226	}
227	puts $outfid $ascmdfVect(newtitle)
228	puts $outfid $ascmdfVect(newminus)
229	}
230	}
231	more {
232	puts $outfid $data
233	}
234	default {
235	error "Unexpected line:\n$data\n in input file."
236	}
237	}
238	}
239	}
240	}
241	}
242	}
243	}
244
245	proc mdf_ascend2excel {infid outfid} {
246	# internal function to strip extra headers from ascend dat files
247	# turns the var definitions on their sides as column titles in a row
248	# after putting out the row of units. drops the dataset headers.
249	global ascmdfVect
250	set state error
251	if {![info exists ascmdfVect(oldvars)]} {
252	set ascmdfVect(oldvars) "{ERROR reading input} {}"
253	}
254	set ok 1
255	while {$ok} {
256	set cc [gets $infid data]
257	if {$cc<0} {break}
258	set line [stringcompact $data]
259	if {[string length $line] <= 0} {
260	continue;
261	}
262	set c [lindex $line 0]
263	switch -exact -- $c {
264	DATASET {
265	set ascmdfVect(newline1) $data; # keep in case next set is new
266	set state header
267	}
268	Observations: -
269	States: {
270	set ascmdfVect(newline2) $data
271	set ascmdfVect(newvars) {}
272	}
273	default {
274	switch -- [string index $line 0] {
275	\{ {
276	lappend ascmdfVect(newvars) $data
277	}
278	default {
279	switch $state {
280	error {
281	error "DATASET not seen as expected in digestion"
282	}
283	header {
284	set state coltitle
285	set ascmdfVect(newtitle) $data
286	}
287	coltitle {
288	set state rowminus
289	set ascmdfVect(newminus) $data
290	}
291	rowminus {
292	set state more
293	if {[string compare \
294	$ascmdfVect(newvars) $ascmdfVect(oldvars)] != 0} {
295	set ascmdfVect(oldvars) $ascmdfVect(newvars)
296	# puts $outfid $ascmdfVect(newline1)
297	# puts $outfid $ascmdfVect(newline2)
298	# write a line of units
299	foreach vdef $ascmdfVect(newvars) {
300	puts -nonewline $outfid "\t[lindex $vdef 2]"
301	}
302	puts $outfid ""
303	# write a line of names
304	foreach vdef $ascmdfVect(newvars) {
305	puts -nonewline $outfid "\t[lindex $vdef 1]"
306	}
307	puts $outfid ""
308	# puts $outfid $ascmdfVect(newtitle)
309	# puts $outfid $ascmdfVect(newminus)
310	}
311	}
312	more {
313	puts $outfid $data
314	}
315	default {
316	error "Unexpected line:\n$data\n in input file."
317	}
318	}
319	}
320	}
321	}
322	}
323	}
324	}
325
326	proc mdf_ascend2matlab {infid outfid} {
327	# internal function to strip extra headers from ascend dat files
328	# and generate a single matlab matrix
329	global ascmdfVect
330	set state error
331	if {![info exists ascmdfVect(oldvars)]} {
332	set ascmdfVect(oldvars) "{ERROR reading input} {}"
333	}
334	set ok 1
335	while {$ok} {
336	set cc [gets $infid data]
337	if {$cc<0} {break}
338	set line [stringcompact $data]
339	if {[string length $line] <= 0} {
340	continue;
341	}
342	set c [lindex $line 0]
343	switch -exact -- $c {
344	DATASET {
345	set ascmdfVect(newline1) $data; # keep in case next set is new
346	set state header
347	}
348	Observations: -
349	States: {
350	set ascmdfVect(newline2) $data
351	set ascmdfVect(newvars) {}
352	}
353	default {
354	switch -- [string index $line 0] {
355	\{ {
356	lappend ascmdfVect(newvars) $data
357	}
358	default {
359	switch $state {
360	error {
361	error "DATASET not seen as expected in digestion"
362	}
363	header {
364	set state coltitle
365	set ascmdfVect(newtitle) $data
366	}
367	coltitle {
368	set state rowminus
369	set ascmdfVect(newminus) $data
370	}
371	rowminus {
372	set state more
373	if {[string compare \
374	$ascmdfVect(newvars) $ascmdfVect(oldvars)] != 0} {
375	if {$ascmdfVect(matlabheaderdone)} {
376	error "Cannot merge unlike data sets into a matlab file"
377	}
378	set ascmdfVect(oldvars) $ascmdfVect(newvars)
379	puts $outfid "%%$ascmdfVect(newline1)"
380	puts $outfid "%%$ascmdfVect(newline2)"
381	foreach vdef $ascmdfVect(newvars) {
382	puts $outfid "%%$vdef"
383	}
384	puts $outfid "%%$ascmdfVect(newtitle)"
385	puts $outfid "%%$ascmdfVect(newminus)"
386	puts $outfid "echo off"
387	puts $outfid $ascmdfVect(matname)var=\[
388	set ascmdfVect(matlabheaderdone) 1
389	} else {
390	puts stderr "DUPLICATE HEADER"
391	}
392	}
393	more {
394	puts $outfid $data
395	}
396	default {
397	error "Unexpected line:\n$data\n in input file."
398	}
399	}
400	}
401	}
402	}
403	}
404	}
405	}