From a1986c5ecb46b39d7a987f1f6ffa625b35d2a54b Mon Sep 17 00:00:00 2001 From: Remington Rohel <77300402+RemingtonRohel@users.noreply.github.com> Date: Tue, 18 Jun 2024 12:39:14 -0600 Subject: [PATCH] Release/v1.0 (#17) First release of borealis_postprocessors. The package is able to reproduce the standard Borealis processing chain, and easily facilitates new processing methods starting from any of the standard Borealis data files (antennas_iq, bfiq, and rawacf). --- LICENSE | 1128 ++++++++++------- README.md | 118 +- data_processing/bfiq_to_rawacf.py | 296 ----- data_processing/convert_base.py | 219 ---- data_processing/utils/restructure.py | 79 -- postprocessors/__init__.py | 22 + .../core/__init__.py | 0 .../core}/antennas_iq_to_bfiq.py | 201 +-- postprocessors/core/antennas_iq_to_rawacf.py | 73 ++ postprocessors/core/bfiq_to_rawacf.py | 285 +++++ .../core/conversion.py | 57 +- postprocessors/core/convert_base.py | 306 +++++ postprocessors/core/restructure.py | 147 +++ .../exceptions}/__init__.py | 0 .../exceptions}/conversion_exceptions.py | 0 .../exceptions/processing_exceptions.py | 36 + .../sandbox/__init__.py | 0 postprocessors/sandbox/binomial_beams.py | 57 + postprocessors/sandbox/bistatic_processing.py | 143 +++ postprocessors/sandbox/extract_beam.py | 109 ++ postprocessors/sandbox/extract_timestamps.py | 83 ++ postprocessors/sandbox/hamming_beams.py | 58 + postprocessors/sandbox/hamming_corrected.py | 152 +++ .../sandbox/hamming_corrected_8_antennas.py | 65 + .../sandbox/process_impt.py | 44 +- postprocessors/sandbox/rawacf_lag0.py | 108 ++ .../sandbox/rawacf_record_averaging.py | 117 ++ .../sandbox/widebeam_antennas_iq_to_bfiq.py | 27 +- .../utils}/__init__.py | 0 postprocessors/utils/filename_conversions.py | 73 ++ pyproject.toml | 26 + scripts/bistatic_postprocessing.py | 125 ++ scripts/memory_limited_postprocessing.py | 85 ++ scripts/timestamp_extraction.py | 73 ++ scripts/widebeam_processing.py | 51 + setup.py | 10 + test/README.md | 8 + test/test_antiq2bfiq.py | 47 +- test/test_antiq2rawacf.py | 47 +- test/test_bfiq2rawacf.py | 47 +- test/utils/compare_files.py | 32 +- 41 files changed, 3226 insertions(+), 1328 deletions(-) delete mode 100644 data_processing/bfiq_to_rawacf.py delete mode 100644 data_processing/convert_base.py delete mode 100644 data_processing/utils/restructure.py create mode 100644 postprocessors/__init__.py rename __init__.py => postprocessors/core/__init__.py (100%) rename {data_processing => postprocessors/core}/antennas_iq_to_bfiq.py (66%) create mode 100644 postprocessors/core/antennas_iq_to_rawacf.py create mode 100644 postprocessors/core/bfiq_to_rawacf.py rename conversion.py => postprocessors/core/conversion.py (67%) mode change 100755 => 100644 create mode 100644 postprocessors/core/convert_base.py create mode 100644 postprocessors/core/restructure.py rename {data_processing => postprocessors/exceptions}/__init__.py (100%) rename {exceptions => postprocessors/exceptions}/conversion_exceptions.py (100%) create mode 100644 postprocessors/exceptions/processing_exceptions.py rename {data_processing => postprocessors}/sandbox/__init__.py (100%) create mode 100644 postprocessors/sandbox/binomial_beams.py create mode 100644 postprocessors/sandbox/bistatic_processing.py create mode 100644 postprocessors/sandbox/extract_beam.py create mode 100644 postprocessors/sandbox/extract_timestamps.py create mode 100644 postprocessors/sandbox/hamming_beams.py create mode 100644 postprocessors/sandbox/hamming_corrected.py create mode 100644 postprocessors/sandbox/hamming_corrected_8_antennas.py rename data_processing/antennas_iq_to_rawacf.py => postprocessors/sandbox/process_impt.py (60%) create mode 100644 postprocessors/sandbox/rawacf_lag0.py create mode 100644 postprocessors/sandbox/rawacf_record_averaging.py rename data_processing/sandbox/widebeam_antennas_iq_to_rawacf.py => postprocessors/sandbox/widebeam_antennas_iq_to_bfiq.py (78%) rename {exceptions => postprocessors/utils}/__init__.py (100%) create mode 100644 postprocessors/utils/filename_conversions.py create mode 100644 pyproject.toml create mode 100644 scripts/bistatic_postprocessing.py create mode 100644 scripts/memory_limited_postprocessing.py create mode 100644 scripts/timestamp_extraction.py create mode 100644 scripts/widebeam_processing.py create mode 100644 setup.py create mode 100644 test/README.md diff --git a/LICENSE b/LICENSE index 8000a6f..f288702 100644 --- a/LICENSE +++ b/LICENSE @@ -1,504 +1,674 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. -[This is the first released version of the Lesser GPL. It also counts - as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.] - Preamble - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some -specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You -can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better -strategy to use in any particular case, based on the explanations below. - - When we speak of free software, we are referring to freedom of use, -not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge -for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of -it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid -distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for -you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them -with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the -library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that -there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know -that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be -introduced by others. - - Finally, software patents pose a constant threat to the existence of -any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a -restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be -consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the -ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and -is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those -libraries into non-free programs. - - When a program is linked with a library, whether statically or using -a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary -General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General -Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it -does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less -of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many -libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to -encourage the widest possible use of a certain library, so that it becomes -a de-facto standard. To achieve this, non-free programs must be -allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this -case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free -programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in -non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating -system. - - Although the Lesser General Public License is Less protective of the -users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run -that program using a modified version of the Library. + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, whereas the latter must -be combined with the library in order to run. - - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other -program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of -this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. - - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also combine or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, - rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if - the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties with +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of this License. - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add -an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus -excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. END OF TERMS AND CONDITIONS - How to Apply These Terms to Your New Libraries + How to Apply These Terms to Your New Programs - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. - To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. - + Copyright (C) - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. - This library is distributed in the hope that it will be useful, + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 - USA + You should have received a copy of the GNU General Public License + along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random - Hacker. - - , 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md index 0afbb8b..92bd649 100644 --- a/README.md +++ b/README.md @@ -7,30 +7,126 @@ The file conversion.py contains the top-level functionality, and is the only fil files. The usage is as follows: ``` -$ python3 conversion.py [-h] infile outfile infile_type outfile_type infile_structure outfile_structure [--averaging-method a] +$ python3 conversion.py [-h] infile outfile infile_type outfile_type infile_structure outfile_structure ``` Pass in the filename you wish to convert, the filename you wish to save as, and the types and structures of both. The script will convert the input file into an output file of type "outfile_type" and structure "outfile_structure". -If the final type is rawacf, the averaging method may optionally be specified as well. If using this package from a python script, the usage is similar. Import `ConvertFile` into your script, then initialize an instance of `ConvertFile` and all the conversion and processing will be done automatically. The usage for this method is as follows: ```python3 -from conversion import ConvertFile -ConvertFile(infile, outfile, infile_type, outfile_type, infile_structure, outfile_structure, - averaging_method=averaging_method) +import postprocessors as pp + +# define infile, outfile, infile_type, outfile_type, infile_structure, and outfile_structure here +pp.ConvertFile(infile, outfile, infile_type, outfile_type, infile_structure, outfile_structure) +``` + +### Defining your own processing method +Following the template of the core processing chains (antennas_iq to bfiq, bfiq to rawacf, antennas_iq to rawacf), +it is fairly straightforward to define your own class for processing files. It is recommended you make this class +within the `/postprocessors/sandbox/` directory. Your class must have a few key components, following the template +below: + +```python3 +from collections import OrderedDict +from postprocessors import BaseConvert + + +class CustomProcessing(BaseConvert): + """ + Custom class for processing SuperDARN Borealis files. + """ + # You can add class variables here, that will be accessible within process_record(). + + def __init__(self, infile: str, outfile: str, infile_type: str, outfile_type: str, infile_structure: str, + outfile_structure: str): + """ + Feel free to add more parameters to this, or hard-code in the file types and structures if you don't need + them to be variable. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_type: str + Borealis data type of input file. One of 'antennas_iq', 'bfiq', or 'rawacf'. + outfile_type: str + Borealis data type of output file. One of 'antennas_iq', 'bfiq', or 'rawacf'. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + # This call will check if infile is `array` structured, and if so, restructure it to `site` structure. + # This ensures that all data from an integration time is collected into one 'record', which is passed + # to process_record() within process_file() + super().__init__(infile, outfile, infile_type, outfile_type, infile_structure, outfile_structure) + + @classmethod + def process_record(cls, record: OrderedDict, **kwargs) -> OrderedDict: + """ + Takes a record from a data file and processes it. + This method is called from within self.process_file() above, and handles the processing of one record + from a site-structured Borealis file. It is up to you to verify that this processing is handled correctly. + + Parameters + ---------- + record: OrderedDict + hdf5 group containing data and metadata, from infile + kwargs: dict + Whatever you need to do your processing! + + Returns + ------- + record: OrderedDict + Whatever you want to store in your file! + """ + # do your processing here + return record ``` +To process a file with your CustomProcessing class, you can do the following within a python script: + +```python3 +from postprocessors.sandbox.custom_processing import CustomProcessing + +# define infile, outfile, infile_type, outfile_type, infile_structure, and outfile_structure here +processor = CustomProcessing(infile, outfile, infile_type, outfile_type, infile_structure, outfile_structure) +processor.process_file() +``` + +If you would like to use multiprocessing to speed up the job, you can pass e.g. `num_processes=5` to spread the job +over 5 cores of your machine. If you are doing so, be sure that the main python script runs in a `if __name__ == '__main__':` block +so the multiprocessing works as expected. + ## Package Structure -### data_processing +### postprocessors Contains the data processing chain, for all processing stages. -The main classes in this module are BaseConvert, ProcessAntennasIQ2Bfiq, ProcessBfiq2Rawacf, and -ProcessAntennasIQ2Rawacf. These classes handle their respective processing between data levels. Additionally, -BaseConvert handles restructuring between file structures, using pyDARNio. -### exceptions -Contains all exceptions that the package will throw. \ No newline at end of file +#### core +The main classes in this module are BaseConvert, AntennasIQ2Bfiq, Bfiq2Rawacf, AntennasIQ2Rawacf, and ConvertFile. +These classes handle their respective processing between data levels. Additionally, BaseConvert handles restructuring between file structures, using pyDARNio. + +#### exceptions +Contains all exceptions that the package will throw. + +#### sandbox +Extra processing classes for doing non-standard processing of files. This is the place to add new classes +for processing any non-standard experiments that you may run. + +#### utils +Some handy utility functions for processing. Currently, the only functions are for generating filenames for downstream +data files. + +### scripts +This directory houses scripts for calling the processing within the `postprocessors` package. +If you plan on batch-processing some data, it is recommended you create your script here. + +### test +This directory houses test scripts and data for verifying the core processing functions of this package. diff --git a/data_processing/bfiq_to_rawacf.py b/data_processing/bfiq_to_rawacf.py deleted file mode 100644 index 0a9de2f..0000000 --- a/data_processing/bfiq_to_rawacf.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright 2021 SuperDARN Canada, University of Saskatchewan -# Author: Marci Detwiller, Remington Rohel -""" -This file contains functions for converting bfiq files -to rawacf files. -""" -import logging -import numpy as np -from collections import OrderedDict -from typing import Union - -from data_processing.convert_base import BaseConvert - -try: - import cupy as xp -except ImportError: - import numpy as xp - cupy_available = False -else: - cupy_available = True - -postprocessing_logger = logging.getLogger('borealis_postprocessing') - - -class ProcessBfiq2Rawacf(BaseConvert): - """ - Class for conversion of Borealis bfiq files into rawacf files. This class inherits from - BaseConvert, which handles all functionality generic to postprocessing borealis files. - - See Also - -------- - ConvertFile - BaseConvert - ProcessAntennasIQ2Bfiq - ProcessAntennasIQ2Rawacf - - Attributes - ---------- - infile: str - The filename of the input antennas_iq file. - outfile: str - The file name of output file - infile_structure: str - The write structure of the file. Structures include: - 'array' - 'site' - outfile_structure: str - The desired structure of the output file. Same structures as - above, with the addition of 'dmap'. - averaging_method: str - Averaging method for computing correlations (for processing into rawacf files). - Acceptable values are 'mean' and 'median'. - """ - - def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, - averaging_method: str = 'mean'): - """ - Initialize the attributes of the class. - - Parameters - ---------- - infile: str - Path to input file. - outfile: str - Path to output file. - infile_structure: str - Borealis structure of input file. Either 'array' or 'site'. - outfile_structure: str - Borealis structure of output file. Either 'array', 'site', or 'dmap'. - averaging_method: str - Method for averaging correlations across sequences. Either 'median' or 'mean'. - """ - super().__init__(infile, outfile, 'bfiq', 'rawacf', infile_structure, outfile_structure) - self.averaging_method = averaging_method - - self.process_file() - - @staticmethod - def process_record(record: OrderedDict, averaging_method: Union[None, str], **kwargs) -> OrderedDict: - """ - Takes a record from a bfiq file and processes it into record for rawacf file. - - Parameters - ---------- - record: OrderedDict - hdf5 record containing bfiq data and metadata - averaging_method: Union[None, str] - Averaging method to use. Supported methods are 'mean' and 'median'. - - Returns - ------- - record: OrderedDict - record converted to rawacf format - """ - if averaging_method is None: - averaging_method = 'mean' - record['averaging_method'] = averaging_method - - correlations = ProcessBfiq2Rawacf.calculate_correlations(record, averaging_method) - record['main_acfs'] = correlations[0] - record['intf_acfs'] = correlations[1] - record['xcfs'] = correlations[2] - - record['correlation_descriptors'] = ProcessBfiq2Rawacf.get_correlation_descriptors() - record['correlation_dimensions'] = ProcessBfiq2Rawacf.get_correlation_dimensions(record) - record = ProcessBfiq2Rawacf.remove_extra_fields(record) - - return record - - @staticmethod - def calculate_correlations(record: OrderedDict, averaging_method: str) -> tuple: - """ - Calculates the auto- and cross-correlations for main and interferometer arrays given the bfiq data in record. - - Parameters - ---------- - record: OrderedDict - hdf5 record containing bfiq data and metadata - averaging_method: str - Averaging method. Supported types are 'mean' and 'median' - - Returns - ------- - main_acfs: np.array - Autocorrelation of the main array data - intf_acfs: np.array - Autocorrelation of the interferometer array data - xcfs: np.array - Cross-correlation of the main and interferometer arrays - """ - # TODO: Figure out how to remove pulse offsets - pulse_phase_offset = record['pulse_phase_offset'] - - # bfiq data shape = [num_arrays, num_sequences, num_beams, num_samps] - bfiq_data = record['data'] - - # Get the data and reshape - num_arrays, num_sequences, num_beams, num_samps = record['data_dimensions'] - bfiq_data = bfiq_data.reshape(record['data_dimensions']) - - num_lags = len(record['lags']) - main_corrs_unavg = xp.zeros((num_sequences, num_beams, record['num_ranges'], num_lags), dtype=xp.complex64) - intf_corrs_unavg = xp.zeros((num_sequences, num_beams, record['num_ranges'], num_lags), dtype=xp.complex64) - cross_corrs_unavg = xp.zeros((num_sequences, num_beams, record['num_ranges'], num_lags), dtype=xp.complex64) - - # Loop through every sequence and compute correlations. - # Output shape after loop is [num_sequences, num_beams, num_range_gates, num_lags] - for sequence in range(num_sequences): - # data input shape = [num_antenna_arrays, num_beams, num_samps] - # data return shape = [num_beams, num_range_gates, num_lags] - main_corrs_unavg[sequence, ...] = \ - ProcessBfiq2Rawacf.correlations_from_samples(bfiq_data[0, sequence, :, :], - bfiq_data[0, sequence, :, :], - record) - intf_corrs_unavg[sequence, ...] = \ - ProcessBfiq2Rawacf.correlations_from_samples(bfiq_data[1, sequence, :, :], - bfiq_data[1, sequence, :, :], - record) - cross_corrs_unavg[sequence, ...] = \ - ProcessBfiq2Rawacf.correlations_from_samples(bfiq_data[1, sequence, :, :], - bfiq_data[0, sequence, :, :], - record) - - if averaging_method == 'median': - main_corrs = xp.median(xp.real(main_corrs_unavg), axis=0) + 1j * xp.median(xp.imag(main_corrs_unavg), - axis=0) - intf_corrs = xp.median(xp.real(intf_corrs_unavg), axis=0) + 1j * xp.median(xp.imag(intf_corrs_unavg), - axis=0) - cross_corrs = xp.median(xp.real(cross_corrs_unavg), axis=0) + 1j * xp.median(xp.imag(cross_corrs_unavg), - axis=0) - else: - # Using mean averaging - main_corrs = xp.einsum('ijkl->jkl', main_corrs_unavg) / num_sequences - intf_corrs = xp.einsum('ijkl->jkl', intf_corrs_unavg) / num_sequences - cross_corrs = xp.einsum('ijkl->jkl', cross_corrs_unavg) / num_sequences - - main_acfs = main_corrs.flatten() - intf_acfs = intf_corrs.flatten() - xcfs = cross_corrs.flatten() - - return main_acfs, intf_acfs, xcfs - - @staticmethod - def correlations_from_samples(beamformed_samples_1: np.array, beamformed_samples_2: np.array, - record: OrderedDict) -> np.array: - """ - Correlate two sets of beamformed samples together. Correlation matrices are used and - indices corresponding to lag pulse pairs are extracted. - - Parameters - ---------- - beamformed_samples_1: ndarray [num_slices, num_beams, num_samples] - The first beamformed samples. - beamformed_samples_2: ndarray [num_slices, num_beams, num_samples] - The second beamformed samples. - record: OrderedDict - hdf5 record containing bfiq data and metadata - - Returns - ------- - values: np.array - Array of correlations for each beam, range, and lag - """ - - # beamformed_samples_1: [num_beams, num_samples] - # beamformed_samples_2: [num_beams, num_samples] - # correlated: [num_beams, num_samples, num_samples] - correlated = xp.einsum('jk,jl->jkl', beamformed_samples_1, beamformed_samples_2.conj()) - - if cupy_available: - correlated = xp.asnumpy(correlated) - - values = [] - if record['lags'].size == 0: - values.append(xp.array([])) - return values - - # First range offset in samples - sample_off = record['first_range_rtt'] * 1e-6 * record['rx_sample_rate'] - sample_off = xp.int32(sample_off) - - # Helpful values converted to units of samples - range_off = xp.arange(record['num_ranges'], dtype=xp.int32) + sample_off - tau_in_samples = record['tau_spacing'] * 1e-6 * record['rx_sample_rate'] - lag_pulses_as_samples = xp.array(record['lags'], xp.int32) * xp.int32(tau_in_samples) - - # [num_range_gates, 1, 1] - # [1, num_lags, 2] - samples_for_all_range_lags = (range_off[..., xp.newaxis, xp.newaxis] + - lag_pulses_as_samples[xp.newaxis, :, :]) - - # [num_range_gates, num_lags, 2] - row = samples_for_all_range_lags[..., 1].astype(xp.int32) - - # [num_range_gates, num_lags, 2] - column = samples_for_all_range_lags[..., 0].astype(xp.int32) - - # [num_beams, num_range_gates, num_lags] - values = correlated[:, row, column] - - # Find the sample that corresponds to the second pulse transmitting - second_pulse_sample_num = xp.int32(tau_in_samples) * record['pulses'][1] - sample_off - 1 - - # Replace all ranges which are contaminated by the second pulse for lag 0 - # with the data from those ranges after the final pulse. - values[:, second_pulse_sample_num:, 0] = values[:, second_pulse_sample_num:, -1] - - return values - - @staticmethod - def get_correlation_descriptors() -> list: - """ - Returns a list of descriptors corresponding to correlation data dimensions. - """ - return ['num_beams', 'num_ranges', 'num_lags'] - - @staticmethod - def get_correlation_dimensions(record: OrderedDict) -> np.array: - """ - Returns the dimensions of correlation data. - - Parameters - ---------- - record: OrderedDict - hdf5 record containing bfiq data and metadata - - Returns - ------- - Array of ints characterizing the data dimensions - """ - return xp.array([len(record['beam_azms']), record['num_ranges'], len(record['lags'])], dtype=xp.uint32) - - @staticmethod - def remove_extra_fields(record: OrderedDict) -> OrderedDict: - """ - Removes fields not needed by the rawacf data format. - - Parameters - ---------- - record: OrderedDict - hdf5 record containing bfiq data and metadata - - Returns - ------- - record: OrderedDict - hdf5 record without fields that aren't in the rawacf format - """ - record.pop('data') - record.pop('data_descriptors') - record.pop('data_dimensions') - record.pop('num_ranges') - record.pop('num_samps') - record.pop('pulse_phase_offset') - record.pop('antenna_arrays_order') - - return record diff --git a/data_processing/convert_base.py b/data_processing/convert_base.py deleted file mode 100644 index b70df52..0000000 --- a/data_processing/convert_base.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2021 SuperDARN Canada, University of Saskatchewan -# Author: Remington Rohel -""" -This file contains functions for converting antennas_iq files -to bfiq files. -""" -import os -import subprocess as sp -from collections import OrderedDict -from typing import Union -import deepdish as dd - -from data_processing.utils.restructure import restructure, convert_to_numpy, FILE_STRUCTURE_MAPPING -from exceptions import conversion_exceptions - -try: - import cupy as xp -except ImportError: - import numpy as xp - cupy_available = False -else: - cupy_available = True - -import logging - -postprocessing_logger = logging.getLogger('borealis_postprocessing') - - -class BaseConvert(object): - """ - Class for converting Borealis filetypes of all structures. This class abstracts and redirects - the file being converted to the correct class (ProcessAntennasIQ2Bfiq, ProcessAntennasIQ2Rawacf, - or ProcessBfiq2Rawacf). - - See Also - -------- - ProcessAntennasIQ2Bfiq - ProcessAntennasIQ2Rawacf - ProcessBfiq2Rawacf - ConvertFile - - Attributes - ---------- - infile: str - The filename of the input file containing SuperDARN data. - outfile: str - The file name of output file - infile_type: str - Type of data file. Types include: - 'antennas_iq' - 'bfiq' - 'rawacf' - outfile_type: str - Desired type of output data file. Same types as above. - infile_structure: str - The write structure of the file. Structures include: - 'array' - 'site' - 'iqdat' (bfiq only) - 'dmap' (rawacf only) - All borealis files are either 'site' or 'array' structured. - outfile_structure: str - The desired structure of the output file. Same structures as - above. - """ - def __init__(self, infile: str, outfile: str, infile_type: str, outfile_type: str, infile_structure: str, - outfile_structure: str): - """ - Initializes the attributes of the class. - - Parameters - ---------- - infile: str - Path to the input file - outfile: str - Path to the output file - infile_type: str - Borealis file type of input file. Supported types are: - 'antennas_iq' - 'bfiq' - 'rawacf' - outfile_type: str - Borealis file type of output file. Supported types are same as for file_type. - infile_structure: str - Borealis file structure of input file. Supported structures are: - 'array' - 'site' - outfile_structure: str - Borealis file structure of output file. Supported structures are: - 'array' - 'site' - 'iqdat' (bfiq only) - 'dmap' (rawacf only) - """ - self.infile = infile - self.outfile = outfile - self.infile_type = infile_type - self.infile_structure = infile_structure - self.outfile_type = outfile_type - self.outfile_structure = outfile_structure - self.check_args() - - self.averaging_method = None - self._temp_files = [] - - def check_args(self): - - if self.infile_structure not in FILE_STRUCTURE_MAPPING[self.infile_type]: - raise conversion_exceptions.ImproperFileStructureError( - f'Input file structure "{self.infile_structure}" is not compatible with input file type ' - f'"{self.infile_type}": Valid structures for {self.infile_type} are ' - f'{FILE_STRUCTURE_MAPPING[self.infile_type]}' - ) - if self.outfile_structure not in FILE_STRUCTURE_MAPPING[self.outfile_type]: - raise conversion_exceptions.ImproperFileStructureError( - f'Output file structure "{self.outfile_structure}" is not compatible with output file type ' - f'"{self.outfile_type}": Valid structures for {self.outfile_type} are ' - f'{FILE_STRUCTURE_MAPPING[self.outfile_type]}' - ) - if self.infile_structure not in ['array', 'site']: - raise conversion_exceptions.ConversionUpstreamError( - f'Input file structure "{self.infile_structure}" cannot be reprocessed into any other format.' - ) - - def process_file(self, **kwargs): - """ - Applies appropriate downstream processing to convert between file types (for site-structured - files only). The processing chain is as follows: - 1. Restructure to site format - 2. Apply appropriate downstream processing - 3. Restructure to final format - 4. Remove all intermediate files created along the way - - See Also - -------- - ProcessAntennasIQ2Bfiq - ProcessAntennasIQ2Rawacf - ProcessBfiq2Rawacf - """ - try: - # Restructure to 'site' format if necessary - if self.infile_structure != 'site': - file_to_process = f'{self.infile}.site.tmp' - self._temp_files.append(file_to_process) - # Restructure file to site format for processing - postprocessing_logger.info(f'Restructuring file {self.infile} --> {file_to_process}') - restructure(self.infile, file_to_process, self.infile_type, self.infile_structure, 'site') - else: - file_to_process = self.infile - - # Prepare to restructure after processing, if necessary - if self.outfile_structure != 'site': - processed_file = f'{self.outfile}.site.tmp' - self._temp_files.append(processed_file) - else: - processed_file = self.outfile - - postprocessing_logger.info(f'Converting file {file_to_process} --> {processed_file}') - - # Load file - group = dd.io.load(file_to_process) - records = group.keys() - - # Process each record - for record in records: - record_dict = group[record] - beamformed_record = self.process_record(record_dict, self.averaging_method, **kwargs) - - # Convert to numpy arrays for saving to file with deepdish - formatted_record = convert_to_numpy(beamformed_record) - - # Save record to temporary file - tempfile = f'/tmp/{record}.tmp' - dd.io.save(tempfile, formatted_record, compression=None) - - # Copy record to output file - cmd = f'h5copy -i {tempfile} -o {processed_file} -s / -d {record}' - sp.call(cmd.split()) - - # Remove temporary file - os.remove(tempfile) - - # Restructure to final structure format, if necessary - if self.outfile_structure != 'site': - postprocessing_logger.info(f'Restructuring file {processed_file} --> {self.outfile}') - restructure(processed_file, self.outfile, self.outfile_type, 'site', self.outfile_structure) - except (Exception,): - postprocessing_logger.error(f'Could not process file {self.infile} -> {self.outfile}. Removing all newly' - f' generated files.') - finally: - self._remove_temp_files() - - def _remove_temp_files(self): - """ - Deletes all temporary files used in the processing chain. - """ - for filename in self._temp_files: - os.remove(filename) - - @staticmethod - def process_record(record: OrderedDict, averaging_method: Union[None, str], **kwargs) -> OrderedDict: - """ - This method should be overwritten by child classes, and should contain the necessary - steps to process a record of input type to output type. - - Parameters - ---------- - record: OrderedDict - An hdf5 group containing one record of site-structured data - averaging_method: Union[None, str] - Method to use for averaging correlations across sequences. - - Returns - ------- - record: OrderedDict - The same hdf5 group, but with the necessary modifications to conform to the standard - of data for self.final_type. - """ - return record diff --git a/data_processing/utils/restructure.py b/data_processing/utils/restructure.py deleted file mode 100644 index 9d128f5..0000000 --- a/data_processing/utils/restructure.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2021 SuperDARN Canada, University of Saskatchewan -# Author: Remington Rohel - -""" -This module provides some functions which are used in the processing of Borealis data files. -""" -import pydarnio -import numpy as np - - -def restructure(infile_name, outfile_name, infile_type, infile_structure, outfile_structure): - """ - This method restructures filename of structure "file_structure" into "final_structure". - - Parameters - ---------- - infile_name: str - Name of the original file. - outfile_name: str - Name of the restructured file. - infile_type: str - Borealis file type of the files. - infile_structure: str - The current write structure of the file. One of 'array' or 'site'. - outfile_structure: str - The desired write structure of the file. One of 'array', 'site', 'iqdat', or 'dmap'. - """ - # dmap and iqdat are not borealis formats, so they are handled specially - if outfile_structure == 'dmap' or outfile_structure == 'iqdat': - pydarnio.BorealisConvert(infile_name, infile_type, outfile_name, - borealis_file_structure=infile_structure) - return - - # Get data from the file - reader = pydarnio.BorealisRead(infile_name, infile_type, infile_structure) - - # Get data in correct format for writing to output file - if outfile_structure == 'site': - data = reader.records - else: - data = reader.arrays - - # Write to output file - pydarnio.BorealisWrite(outfile_name, data, infile_type, outfile_structure) - - -def convert_to_numpy(data: dict): - """Converts lists stored in dict into numpy array. Recursive. - Args: - data (Python dictionary): Dictionary with lists to convert to numpy arrays. - """ - for k, v in data.items(): - if isinstance(v, dict): - convert_to_numpy(v) - elif isinstance(v, list): - data[k] = np.array(v) - else: - continue - - return data - - -# Dictionary mapping the accepted borealis file types to the borealis file types that they can be processed into. -# The dictionary keys are the valid input file types, and their values are lists of file types which they can be -# processed into. -FILE_TYPE_MAPPING = { - 'antennas_iq': ['antennas_iq', 'bfiq', 'rawacf'], - 'bfiq': ['bfiq', 'rawacf'], - 'rawacf': ['rawacf'] -} - -# Dictionary mapping the accepted borealis file types to the borealis file structures that they can be formatted as. -# The dictionary keys are the valid input file types, and their values are lists of file structures which they can be -# formatted as. -FILE_STRUCTURE_MAPPING = { - 'antennas_iq': ['site', 'array'], - 'bfiq': ['site', 'array', 'iqdat'], - 'rawacf': ['site', 'array', 'dmap'] -} diff --git a/postprocessors/__init__.py b/postprocessors/__init__.py new file mode 100644 index 0000000..092951d --- /dev/null +++ b/postprocessors/__init__.py @@ -0,0 +1,22 @@ +""" +Copyright 2022 SuperDARN Canada + +General imports for borealis_postprocessors project. +""" + +# Importing exception classes +from .exceptions import conversion_exceptions +from .exceptions import processing_exceptions + +# The main processing classes +from .core.convert_base import BaseConvert +from .core.antennas_iq_to_bfiq import AntennasIQ2Bfiq +from .core.bfiq_to_rawacf import Bfiq2Rawacf +from .core.antennas_iq_to_rawacf import AntennasIQ2Rawacf +from .core.conversion import ConvertFile + +# Helpful for scripts +from .utils.filename_conversions import borealis_to_sdarn_rename, borealis_to_borealis_rename + +# This stays minimal, up to user to import further for their usage +from .sandbox import * diff --git a/__init__.py b/postprocessors/core/__init__.py similarity index 100% rename from __init__.py rename to postprocessors/core/__init__.py diff --git a/data_processing/antennas_iq_to_bfiq.py b/postprocessors/core/antennas_iq_to_bfiq.py similarity index 66% rename from data_processing/antennas_iq_to_bfiq.py rename to postprocessors/core/antennas_iq_to_bfiq.py index d9686d8..80b7dcd 100644 --- a/data_processing/antennas_iq_to_bfiq.py +++ b/postprocessors/core/antennas_iq_to_bfiq.py @@ -6,28 +6,18 @@ """ import itertools from collections import OrderedDict -from typing import Union import numpy as np from scipy.constants import speed_of_light -from data_processing.convert_base import BaseConvert - -try: - import cupy as xp -except ImportError: - import numpy as xp - - cupy_available = False -else: - cupy_available = True +from postprocessors import BaseConvert import logging postprocessing_logger = logging.getLogger('borealis_postprocessing') -class ProcessAntennasIQ2Bfiq(BaseConvert): +class AntennasIQ2Bfiq(BaseConvert): """ Class for conversion of Borealis antennas_iq files into bfiq files. This class inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. @@ -36,7 +26,6 @@ class ProcessAntennasIQ2Bfiq(BaseConvert): -------- ConvertFile BaseConvert - ProcessAntennasIQ2Rawacf Attributes ---------- @@ -45,12 +34,13 @@ class ProcessAntennasIQ2Bfiq(BaseConvert): outfile: str The file name of output file infile_structure: str - The write structure of the file. Structures include: + The structure of the file. Structures include: 'array' 'site' outfile_structure: str The desired structure of the output file. Same structures as above, plus 'iqdat'. """ + window = [1.0] * 16 def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str): """ @@ -69,10 +59,8 @@ def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_str """ super().__init__(infile, outfile, 'antennas_iq', 'bfiq', infile_structure, outfile_structure) - self.process_file() - - @staticmethod - def process_record(record: OrderedDict, averaging_method: Union[None, str], **kwargs) -> OrderedDict: + @classmethod + def process_record(cls, record: OrderedDict, **kwargs) -> OrderedDict: """ Takes a record from an antennas_iq file and converts it into a bfiq record. @@ -80,28 +68,26 @@ def process_record(record: OrderedDict, averaging_method: Union[None, str], **kw ---------- record: OrderedDict hdf5 record containing antennas_iq data and metadata - averaging_method: Union[None, str] - Method to use for averaging correlations across sequences. Unused by this method. Returns ------- record: OrderedDict hdf5 record, with new fields required by bfiq data format """ - record['first_range'] = ProcessAntennasIQ2Bfiq.calculate_first_range(record) - record['first_range_rtt'] = ProcessAntennasIQ2Bfiq.calculate_first_range_rtt(record) - record['lags'] = ProcessAntennasIQ2Bfiq.create_lag_table(record) - record['range_sep'] = ProcessAntennasIQ2Bfiq.calculate_range_separation(record) - record['num_ranges'] = ProcessAntennasIQ2Bfiq.get_number_of_ranges(record) - record['data'] = ProcessAntennasIQ2Bfiq.beamform_data(record) - record['data_descriptors'] = ProcessAntennasIQ2Bfiq.get_data_descriptors() - record['data_dimensions'] = ProcessAntennasIQ2Bfiq.get_data_dimensions(record) - record['antenna_arrays_order'] = ProcessAntennasIQ2Bfiq.change_antenna_arrays_order() + record['first_range'] = cls.calculate_first_range(record) + record['first_range_rtt'] = cls.calculate_first_range_rtt(record) + record['lags'] = cls.create_lag_table(record) + record['range_sep'] = cls.calculate_range_separation(record) + record['num_ranges'] = cls.get_number_of_ranges(record) + record['data'] = cls.beamform_data(record) + record['data_descriptors'] = cls.get_data_descriptors() + record['data_dimensions'] = cls.get_data_dimensions(record) + record['antenna_arrays_order'] = cls.change_antenna_arrays_order() return record - @staticmethod - def beamform_data(record: OrderedDict) -> np.array: + @classmethod + def beamform_data(cls, record: OrderedDict) -> np.array: """ Beamforms the data for each array, and stores it back into the record @@ -125,38 +111,45 @@ def beamform_data(record: OrderedDict) -> np.array: num_antennas, num_sequences, num_samps = record['data_dimensions'] antennas_data = antennas_data.reshape(record['data_dimensions']) - main_beamformed_data = xp.array([], dtype=xp.complex64) - intf_beamformed_data = xp.array([], dtype=xp.complex64) + main_beamformed_data = np.array([], dtype=np.complex64) + intf_beamformed_data = np.array([], dtype=np.complex64) main_antenna_count = record['main_antenna_count'] + intf_antenna_count = record['intf_antenna_count'] station = record['station'] main_antenna_spacing = radar_dict[station]['main_antenna_spacing'] intf_antenna_spacing = radar_dict[station]['intf_antenna_spacing'] + githash = record['borealis_git_hash'].split('-')[0].strip('v').split('.') # v0.6.1-xxxxxx -> ['0', '6', '1'] + if githash[0] == '0' and int(githash[1]) < 7: + antenna_indices = np.array([int(i.split('_')[-1]) for i in record['antenna_arrays_order']]) + else: + antenna_indices = np.array([int(i.decode('utf-8').split('_')[-1]) for i in record['antenna_arrays_order']]) + main_antenna_indices = np.array([i for i in antenna_indices if i < main_antenna_count]) + intf_antenna_indices = np.array([i - main_antenna_count for i in antenna_indices if i >= main_antenna_count]) + # Loop through every sequence and beamform the data. # Output shape after loop is [num_sequences, num_beams, num_samps] for sequence in range(num_sequences): # data input shape = [num_antennas, num_samps] # data return shape = [num_beams, num_samps] main_beamformed_data = \ - xp.append(main_beamformed_data, - ProcessAntennasIQ2Bfiq.beamform(antennas_data[:main_antenna_count, sequence, :], - beam_azms, - freq, - main_antenna_spacing)) + np.append(main_beamformed_data, cls.beamform(antennas_data[:len(main_antenna_indices), sequence, :], + beam_azms, freq, main_antenna_count, main_antenna_spacing, + main_antenna_indices)) + intf_beamformed_data = \ - xp.append(intf_beamformed_data, - ProcessAntennasIQ2Bfiq.beamform(antennas_data[main_antenna_count:, sequence, :], - beam_azms, - freq, - intf_antenna_spacing)) + np.append(intf_beamformed_data, cls.beamform(antennas_data[len(main_antenna_indices):, sequence, :], + beam_azms, freq, intf_antenna_count, intf_antenna_spacing, + intf_antenna_indices)) - all_data = xp.append(main_beamformed_data, intf_beamformed_data).flatten() + all_data = np.append(main_beamformed_data, intf_beamformed_data).flatten() return all_data - @staticmethod - def beamform(antennas_data: np.array, beamdirs: np.array, rxfreq: float, antenna_spacing: float) -> np.array: + @classmethod + def beamform(cls, antennas_data: np.array, beamdirs: np.array, rxfreq: float, ants_in_array: int, antenna_spacing: float, + antenna_indices: np.array) -> np.array: """ Beamforms the data from each antenna and sums to create one dataset for each beam direction. @@ -169,8 +162,12 @@ def beamform(antennas_data: np.array, beamdirs: np.array, rxfreq: float, antenna Azimuthal beam directions in degrees off boresight rxfreq: float Frequency of the received beam + ants_in_array: int + Number of physical antennas in the array antenna_spacing: float Spacing in metres between antennas (assumed uniform) + antenna_indices: np.array + Mapping of antenna channels to physical antennas in the uniformly spaced array. Returns ------- @@ -187,30 +184,29 @@ def beamform(antennas_data: np.array, beamdirs: np.array, rxfreq: float, antenna antenna_phase_shifts = [] # Get phase shift for each antenna - for antenna in range(num_antennas): - phase_shift = ProcessAntennasIQ2Bfiq.get_phshift(beam_direction, - rxfreq, - antenna, - num_antennas, - antenna_spacing) + for antenna in antenna_indices: + phase_shift = cls.get_phshift(beam_direction, rxfreq, antenna, ants_in_array, antenna_spacing) # Bring into range (-2*pi, 2*pi) - phase_shift = xp.fmod(phase_shift, 2 * xp.pi) antenna_phase_shifts.append(phase_shift) # Apply phase shift to data from respective antenna - phased_antenna_data = [ProcessAntennasIQ2Bfiq.shift_samples(antennas_data[i], antenna_phase_shifts[i], 1.0) - for i in range(num_antennas)] - phased_antenna_data = xp.array(phased_antenna_data) + if num_antennas == 16: + phased_antenna_data = [cls.shift_samples(antennas_data[i], antenna_phase_shifts[i], cls.window[i]) + for i in range(num_antennas)] + else: + phased_antenna_data = [cls.shift_samples(antennas_data[i], antenna_phase_shifts[i], 1.0) + for i in range(num_antennas)] + phased_antenna_data = np.array(phased_antenna_data) # Sum across antennas to get beamformed data - one_beam_data = xp.sum(phased_antenna_data, axis=0) + one_beam_data = np.sum(phased_antenna_data, axis=0) beamformed_data.append(one_beam_data) - beamformed_data = xp.array(beamformed_data) + beamformed_data = np.array(beamformed_data) return beamformed_data - @staticmethod - def get_phshift(beamdir: float, freq: float, antenna: int, num_antennas: int, antenna_spacing: float, + @classmethod + def get_phshift(cls, beamdir: float, freq_khz: float, antenna: int, num_antennas: int, antenna_spacing: float, centre_offset: int = 0.0) -> float: """ Find the phase shift for a given antenna and beam direction. @@ -223,7 +219,7 @@ def get_phshift(beamdir: float, freq: float, antenna: int, num_antennas: int, an beamdir: float The azimuthal direction of the beam off boresight, in degrees, positive beamdir being to the right of the boresight (looking along boresight from ground). This is for this antenna. - freq: float + freq_khz: float Transmit frequency in kHz antenna: int Antenna number, INDEXED FROM ZERO, zero being the leftmost antenna if looking down the boresight @@ -242,22 +238,29 @@ def get_phshift(beamdir: float, freq: float, antenna: int, num_antennas: int, an phshift: float A phase shift for the samples for this antenna number, in radians. """ - freq = freq * 1000.0 # convert to Hz. + freq_hz = freq_khz * 1000.0 # convert to Hz + k = 2 * np.pi * freq_hz / speed_of_light # 2pi / wavelength - # Convert to radians - beamrad = xp.pi * xp.float64(beamdir) / 180.0 + # Convert to radians CW of boresight direction + beamrad = np.pi * np.float64(beamdir) / 180.0 - # Pointing to right of boresight, use point in middle (hypothetically antenna 7.5) as phshift=0 - phshift = 2 * xp.pi * freq * (((num_antennas - 1) / 2.0 - antenna) * antenna_spacing + centre_offset) * \ - xp.cos(xp.pi / 2.0 - beamrad) / speed_of_light + # Middle of array is origin, boresight is in +y direction, so antenna=0 is leftmost antenna and -ve position + # when looking down +y axis. + # E.g., for 16 antennas: + # antenna = 0 -> antenna_idx = -7.5, + # antenna = 1 -> antenna_idx = -6.5, + # ... + # antenna = 15 -> antenna_idx = 7.5 + antenna_idx = antenna - (num_antennas - 1) / 2.0 + antenna_position = antenna_idx * antenna_spacing + centre_offset - # Bring into range (-2*pi, 2*pi) - phshift = xp.fmod(phshift, 2 * xp.pi) + # phshift = 0 at origin + phshift = -1 * k * antenna_position * np.sin(beamrad) return phshift - @staticmethod - def shift_samples(basic_samples: np.array, phshift: float, amplitude: float = 1.) -> np.array: + @classmethod + def shift_samples(cls, basic_samples: np.array, phshift: float, amplitude: float = 1.) -> np.array: """ Shift samples for a pulse by a given phase shift. Take the samples and shift by given phase shift in rads and adjust amplitude as @@ -277,12 +280,12 @@ def shift_samples(basic_samples: np.array, phshift: float, amplitude: float = 1. samples: np.array Basic_samples that have been shaped for the antenna for the desired beam. """ - samples = amplitude * xp.exp(1j * phshift) * basic_samples + samples = amplitude * np.exp(1j * phshift) * basic_samples return samples - @staticmethod - def calculate_first_range(record: OrderedDict) -> float: + @classmethod + def calculate_first_range(cls, record: OrderedDict) -> float: """ Calculates the distance from the main array to the first range (in km). @@ -296,14 +299,14 @@ def calculate_first_range(record: OrderedDict) -> float: first_range: float Distance to first range in km """ - # TODO: Get this from somewhere, probably linked to the experiment ran. Might need to look up + # TODO: Get this from somewhere, probably linked to the enperiment ran. Might need to look up # based on githash first_range = 180.0 # scf.FIRST_RANGE - return xp.float32(first_range) + return np.float32(first_range) - @staticmethod - def calculate_first_range_rtt(record: OrderedDict) -> float: + @classmethod + def calculate_first_range_rtt(cls, record: OrderedDict) -> float: """ Calculates the round-trip time (in microseconds) to the first range in a record. @@ -320,10 +323,10 @@ def calculate_first_range_rtt(record: OrderedDict) -> float: # km * (there and back) * (km to meters) * (seconds to us) / c first_range_rtt = record['first_range'] * 2.0 * 1.0e3 * 1e6 / speed_of_light - return xp.float32(first_range_rtt) + return np.float32(first_range_rtt) - @staticmethod - def create_lag_table(record: OrderedDict) -> np.array: + @classmethod + def create_lag_table(cls, record: OrderedDict) -> np.array: """ Creates the lag table for the record. @@ -345,12 +348,12 @@ def create_lag_table(record: OrderedDict) -> np.array: lag_table.append([record['pulses'][0], record['pulses'][0]]) # lag 0 lag_table = sorted(lag_table, key=lambda x: x[1] - x[0]) # sort by lag number lag_table.append([record['pulses'][-1], record['pulses'][-1]]) # alternate lag 0 - lags = xp.array(lag_table, dtype=xp.uint32) + lags = np.array(lag_table, dtype=np.uint32) return lags - @staticmethod - def calculate_range_separation(record: OrderedDict) -> float: + @classmethod + def calculate_range_separation(cls, record: OrderedDict) -> float: """ Calculates the separation between ranges in km. @@ -367,10 +370,10 @@ def calculate_range_separation(record: OrderedDict) -> float: # (1 / (sample rate)) * c / (km to meters) / 2 range_sep = 1 / record['rx_sample_rate'] * speed_of_light / 1.0e3 / 2.0 - return xp.float32(range_sep) + return np.float32(range_sep) - @staticmethod - def get_number_of_ranges(record: OrderedDict) -> int: + @classmethod + def get_number_of_ranges(cls, record: OrderedDict) -> int: """ Gets the number of ranges for the record. @@ -385,16 +388,16 @@ def get_number_of_ranges(record: OrderedDict) -> int: The number of ranges of the data """ # Infer the number of ranges from the record metadata - first_range_offset = ProcessAntennasIQ2Bfiq.calculate_first_range_rtt(record) * 1e-6 * record['rx_sample_rate'] - num_ranges = record['num_samps'] - xp.int32(first_range_offset) - record['blanked_samples'][-1] + first_range_offset = cls.calculate_first_range_rtt(record) * 1e-6 * record['rx_sample_rate'] + num_ranges = record['num_samps'] - np.int32(first_range_offset) - record['blanked_samples'][-1] # 3 extra samples taken for each record (not sure why) num_ranges = num_ranges - 3 - return xp.uint32(num_ranges) + return np.uint32(num_ranges) - @staticmethod - def get_data_descriptors() -> list: + @classmethod + def get_data_descriptors(cls) -> list: """ Returns the proper data descriptors for a bfiq file @@ -407,8 +410,8 @@ def get_data_descriptors() -> list: return new_descriptors - @staticmethod - def get_data_dimensions(record: OrderedDict): + @classmethod + def get_data_dimensions(cls, record: OrderedDict): """ Returns a list of the new data dimensions for a bfiq record. @@ -426,13 +429,13 @@ def get_data_dimensions(record: OrderedDict): # New dimensions: [num_antenna_arrays, num_sequences, num_beams, num_samps] old_dimensions = record['data_dimensions'] - new_dimensions = xp.array([2, old_dimensions[1], len(record['beam_azms']), old_dimensions[2]], - dtype=xp.uint32) + new_dimensions = np.array([2, old_dimensions[1], len(record['beam_azms']), old_dimensions[2]], + dtype=np.uint32) return new_dimensions - @staticmethod - def change_antenna_arrays_order() -> list: + @classmethod + def change_antenna_arrays_order(cls) -> list: """ Returns the correct field 'antenna_arrays_order' for a bfiq file @@ -453,5 +456,7 @@ def change_antenna_arrays_order() -> list: 'rkn': {'main_antenna_spacing': 15.24, 'intf_antenna_spacing': 15.24}, 'inv': {'main_antenna_spacing': 15.24, + 'intf_antenna_spacing': 15.24}, + 'lab': {'main_antenna_spacing': 15.24, 'intf_antenna_spacing': 15.24} } diff --git a/postprocessors/core/antennas_iq_to_rawacf.py b/postprocessors/core/antennas_iq_to_rawacf.py new file mode 100644 index 0000000..e7d13a1 --- /dev/null +++ b/postprocessors/core/antennas_iq_to_rawacf.py @@ -0,0 +1,73 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan +# Author: Remington Rohel +""" +This file contains functions for converting antennas_iq files +to rawacf files. +""" +from collections import OrderedDict + +from postprocessors import BaseConvert, AntennasIQ2Bfiq, Bfiq2Rawacf + + +class AntennasIQ2Rawacf(AntennasIQ2Bfiq, Bfiq2Rawacf): + """ + Class for conversion of Borealis antennas_iq files into rawacf files. This class inherits from + BaseConvert, which handles all functionality generic to postprocessing borealis files. + + See Also + -------- + ConvertFile + BaseConvert + AntennasIQ2Bfiq + Bfiq2Rawacf + + Attributes + ---------- + infile: str + The filename of the input antennas_iq file. + outfile: str + The file name of output file + infile_structure: str + The structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + BaseConvert.__init__(self, infile, outfile, 'antennas_iq', 'rawacf', infile_structure, outfile_structure) + + @classmethod + def process_record(cls, record: OrderedDict, **kwargs) -> OrderedDict: + """ + Takes a record from an antennas_iq file process into a rawacf record. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing antennas_iq data and metadata + + Returns + ------- + record: OrderedDict + hdf5 record, with new fields required by rawacf data format + """ + record = super(AntennasIQ2Rawacf, cls).process_record(record, **kwargs) # Calls AntennasIQ2Bfiq.process_record() + record = super(AntennasIQ2Bfiq, cls).process_record(record, **kwargs) # Calls Bfiq2Rawacf.process_record() + + return record diff --git a/postprocessors/core/bfiq_to_rawacf.py b/postprocessors/core/bfiq_to_rawacf.py new file mode 100644 index 0000000..79935f9 --- /dev/null +++ b/postprocessors/core/bfiq_to_rawacf.py @@ -0,0 +1,285 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan +# Author: Marci Detwiller, Remington Rohel +""" +This file contains functions for converting bfiq files +to rawacf files. +""" +import logging +import numpy as np +from collections import OrderedDict + +from postprocessors import BaseConvert + +postprocessing_logger = logging.getLogger('borealis_postprocessing') + + +class Bfiq2Rawacf(BaseConvert): + """ + Class for conversion of Borealis bfiq files into rawacf files. This class inherits from + BaseConvert, which handles all functionality generic to postprocessing borealis files. + + See Also + -------- + ConvertFile + BaseConvert + + Attributes + ---------- + infile: str + The filename of the input antennas_iq file. + outfile: str + The file name of output file + infile_structure: str + The structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as + above, with the addition of 'dmap'. + """ + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, 'bfiq', 'rawacf', infile_structure, outfile_structure) + + @classmethod + def process_record(cls, record: OrderedDict, **kwargs) -> OrderedDict: + """ + Takes a record from a bfiq file and processes it into record for rawacf file. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing bfiq data and metadata + + Returns + ------- + record: OrderedDict + record converted to rawacf format + """ + averaging_method = kwargs.get('averaging_method', 'mean') + record['averaging_method'] = averaging_method + + correlations = cls.calculate_correlations(record, averaging_method) + record['main_acfs'] = correlations[0] + record['intf_acfs'] = correlations[1] + record['xcfs'] = correlations[2] + + githash = record['borealis_git_hash'].split('-')[0].strip('v').split('.') # v0.6.1-xxxxxx -> ['0', '6', '1'] + if githash[0] == '0' and int(githash[1]) < 7: + record['correlation_descriptors'] = cls.get_correlation_descriptors() + record['correlation_dimensions'] = cls.get_correlation_dimensions(record) + else: + record['data_descriptors'] = np.bytes_(cls.get_correlation_descriptors()) + record['data_dimensions'] = cls.get_correlation_dimensions(record) + record = cls.remove_extra_fields(record) + + return record + + @classmethod + def calculate_correlations(cls, record: OrderedDict, averaging_method: str) -> tuple: + """ + Calculates the auto- and cross-correlations for main and interferometer arrays given the bfiq data in record. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing bfiq data and metadata + averaging_method: str + Averaging method. Supported types are 'mean' and 'median' + + Returns + ------- + main_acfs: np.array + Autocorrelation of the main array data + intf_acfs: np.array + Autocorrelation of the interferometer array data + xcfs: np.array + Cross-correlation of the main and interferometer arrays + """ + # bfiq data shape = [num_arrays, num_sequences, num_beams, num_samps] + bfiq_data = record['data'] + + # Get the data and reshape + num_arrays, num_sequences, num_beams, num_samps = record['data_dimensions'] + bfiq_data = bfiq_data.reshape(record['data_dimensions']) + + main_corrs_unavg = cls.correlations_from_samples(bfiq_data[0, ...], bfiq_data[0, ...], record) + intf_corrs_unavg = cls.correlations_from_samples(bfiq_data[1, ...], bfiq_data[1, ...], record) + cross_corrs_unavg = cls.correlations_from_samples(bfiq_data[1, ...], bfiq_data[0, ...], record) + + if averaging_method == 'median': + main_corrs = np.median(np.real(main_corrs_unavg), axis=0) + 1j * np.median(np.imag(main_corrs_unavg), + axis=0) + intf_corrs = np.median(np.real(intf_corrs_unavg), axis=0) + 1j * np.median(np.imag(intf_corrs_unavg), + axis=0) + cross_corrs = np.median(np.real(cross_corrs_unavg), axis=0) + 1j * np.median(np.imag(cross_corrs_unavg), + axis=0) + else: + # Using mean averaging + main_corrs = np.einsum('ijkl->jkl', main_corrs_unavg) / num_sequences + intf_corrs = np.einsum('ijkl->jkl', intf_corrs_unavg) / num_sequences + cross_corrs = np.einsum('ijkl->jkl', cross_corrs_unavg) / num_sequences + + main_acfs = main_corrs + intf_acfs = intf_corrs + xcfs = cross_corrs + + return main_acfs, intf_acfs, xcfs + + @classmethod + def correlations_from_samples(cls, beamformed_samples_1: np.array, beamformed_samples_2: np.array, + record: OrderedDict) -> np.array: + """ + Correlate two sets of beamformed samples together. Correlation matrices are used and + indices corresponding to lag pulse pairs are extracted. + + Parameters + ---------- + beamformed_samples_1: ndarray [num_sequences, num_beams, num_samples] + The first beamformed samples. + beamformed_samples_2: ndarray [num_sequences, num_beams, num_samples] + The second beamformed samples. + record: OrderedDict + hdf5 record containing bfiq data and metadata + + Returns + ------- + values: np.array [num_sequences, num_beams, num_ranges, num_lags] + Array of correlations for each sequence, beam, range, and lag + """ + + values = [] + if record['lags'].size == 0: + values.append(np.array([])) + return values + + num_sequences = beamformed_samples_1.shape[0] + pulses = list(record['pulses']) + pulse_phase_offsets = record['pulse_phase_offset'] + ppo_flag = False + if len(pulse_phase_offsets) != len(record['pulses']): + if len(pulse_phase_offsets) > 1: + if not np.isnan(pulse_phase_offsets[0]): + pulse_phase_offsets = pulse_phase_offsets.reshape((num_sequences, len(record['pulses']))) + ppo_flag = True + + # First range offset in samples + sample_off = record['first_range_rtt'] * 1e-6 * record['rx_sample_rate'] + sample_off = np.int32(sample_off) + + # Helpful values converted to units of samples + range_off = np.arange(record['num_ranges'], dtype=np.int32) + sample_off + tau_in_samples = record['tau_spacing'] * 1e-6 * record['rx_sample_rate'] + lag_pulses_as_samples = np.array(record['lags'], np.int32) * np.int32(tau_in_samples) + + # [num_range_gates, 1, 1] + # [1, num_lags, 2] + samples_for_all_range_lags = (range_off[..., np.newaxis, np.newaxis] + + lag_pulses_as_samples[np.newaxis, :, :]) + + # [num_range_gates, num_lags, 2] + row = samples_for_all_range_lags[..., 1].astype(np.int32) + + # [num_range_gates, num_lags, 2] + column = samples_for_all_range_lags[..., 0].astype(np.int32) + + # [num_sequences, num_beams, num_range_gates, num_lags] + values = np.zeros(beamformed_samples_1.shape[:2] + row.shape[:2], dtype=np.complex64) + + # Find the correlations + for lag in range(row.shape[1]): + values[..., lag] = beamformed_samples_1[..., row[:, lag]] * beamformed_samples_2[..., column[:, lag]].conj() + + # Remove pulse_phase_offsets if they are present + if len(pulse_phase_offsets) == len(pulses): + # The indices in record['pulses'] of the pulses in each lag pair + # [num_lags] + lag1_indices = [pulses.index(val) for val in record['lags'][:, 0]] + lag2_indices = [pulses.index(val) for val in record['lags'][:, 1]] + + # phase offset of first pulse - phase offset of second pulse, for all lag pairs + # [num_lags] + angle_offsets = [np.radians(np.float32(pulse_phase_offsets[lag1_indices[i]]) - + np.float32(pulse_phase_offsets[lag2_indices[i]])) + for i in range(len(lag1_indices))] + + # [num_lags] + phase_offsets = np.exp(1j * np.array(angle_offsets, np.float32)) + + values = np.einsum('ijkl,l->ijkl', values, phase_offsets) + elif len(pulse_phase_offsets) != 0 and ppo_flag: + raise ValueError('Dimensions of pulse_phase_offsets does not match dimensions of pulses') + + # Find the sample that corresponds to the second pulse transmitting + second_pulse_sample_num = np.int32(tau_in_samples) * record['pulses'][1] - sample_off - 1 + + # Replace all ranges which are contaminated by the second pulse for lag 0 + # with the data from those ranges after the final pulse. + values[..., second_pulse_sample_num:, 0] = values[..., second_pulse_sample_num:, -1] + + return values + + @classmethod + def get_correlation_descriptors(cls) -> list: + """ + Returns a list of descriptors corresponding to correlation data dimensions. + """ + return ['num_beams', 'num_ranges', 'num_lags'] + + @classmethod + def get_correlation_dimensions(cls, record: OrderedDict) -> np.array: + """ + Returns the dimensions of correlation data. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing bfiq data and metadata + + Returns + ------- + Array of ints characterizing the data dimensions + """ + return np.array([len(record['beam_azms']), record['num_ranges'], len(record['lags'])], dtype=np.uint32) + + @classmethod + def remove_extra_fields(cls, record: OrderedDict) -> OrderedDict: + """ + Removes fields not needed by the rawacf data format. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing bfiq data and metadata + + Returns + ------- + record: OrderedDict + hdf5 record without fields that aren't in the rawacf format + """ + record.pop('data') + + githash = record['borealis_git_hash'].split('-')[0].strip('v').split('.') # v0.6.1-xxxxxx -> ['0', '6', '1'] + if githash[0] == '0' and int(githash[1]) < 7: + record.pop('data_descriptors') + record.pop('data_dimensions') + record.pop('num_ranges') + record.pop('num_samps') + record.pop('pulse_phase_offset') + record.pop('antenna_arrays_order') + + return record diff --git a/conversion.py b/postprocessors/core/conversion.py old mode 100755 new mode 100644 similarity index 67% rename from conversion.py rename to postprocessors/core/conversion.py index 27992fb..8a72157 --- a/conversion.py +++ b/postprocessors/core/conversion.py @@ -8,46 +8,26 @@ import argparse import os -from data_processing.antennas_iq_to_bfiq import ProcessAntennasIQ2Bfiq -from data_processing.antennas_iq_to_rawacf import ProcessAntennasIQ2Rawacf -from data_processing.bfiq_to_rawacf import ProcessBfiq2Rawacf -from data_processing.convert_base import BaseConvert -from data_processing.utils.restructure import FILE_TYPE_MAPPING, restructure -from exceptions import conversion_exceptions - - -def usage_msg(): - """ - Return the usage message for this process. - This is used if a -h flag or invalid arguments are provided. - """ - - usage_message = """ conversion.py [-h] infile outfile infile_type outfile_type infile_structure outfile_structure [--averaging-method a] - - Pass in the filename you wish to convert, the filename you wish to save as, and the types and structures of both. - The script will convert the input file into an output file of type "outfile_type" and structure "outfile_structure". - If the final type is rawacf, the averaging method may optionally be specified as well. """ - - return usage_message +from postprocessors import AntennasIQ2Bfiq, Bfiq2Rawacf, AntennasIQ2Rawacf, BaseConvert +from postprocessors.core.restructure import FILE_TYPE_MAPPING, restructure +from postprocessors import conversion_exceptions def conversion_parser(): - parser = argparse.ArgumentParser(usage=usage_msg()) + parser = argparse.ArgumentParser() parser.add_argument("infile", help="Path to the file that you wish to convert. (e.g. 20190327.2210.38.sas.0.bfiq.hdf5.site)") parser.add_argument("outfile", help="Path to the location where the output file should be stored. " "(e.g. 20190327.2210.38.sas.0.rawacf.hdf5.site)") - parser.add_argument("infile-type", choices=['antennas_iq', 'bfiq', 'rawacf'], + parser.add_argument("infile_type", metavar="infile-type", choices=['antennas_iq', 'bfiq', 'rawacf'], help="Type of input file.") - parser.add_argument("outfile-type", choices=['antennas_iq', 'bfiq', 'rawacf'], + parser.add_argument("outfile_type", metavar="outfile-type", choices=['antennas_iq', 'bfiq', 'rawacf'], help="Type of output file.") - parser.add_argument("infile-structure", choices=['array', 'site'], + parser.add_argument("infile_structure", metavar="infile-structure", choices=['array', 'site'], help="Structure of input file.") - parser.add_argument("outfile-structure", choices=['array', 'site', 'iqdat', 'dmap'], + parser.add_argument("outfile_structure", metavar="outfile-structure", choices=['array', 'site', 'iqdat', 'dmap'], help="Structure of output file.") - parser.add_argument("-a", "--averaging-method", required=False, default='mean', choices=['mean', 'median'], - help="Averaging method for generating rawacf type file. Default mean.") return parser @@ -80,7 +60,7 @@ class ConvertFile(object): outfile_type: str Desired type of output data file. Same types as above. infile_structure: str - The write structure of the file. Structures include: + The structure of the file. Structures include: 'array' 'site' outfile_structure: str @@ -89,20 +69,16 @@ class ConvertFile(object): 'site' 'iqdat' (bfiq only) 'dmap' (rawacf only) - averaging_method: str - Averaging method for computing correlations (for processing into rawacf files). - Acceptable values are 'mean' and 'median'. """ def __init__(self, infile: str, outfile: str, infile_type: str, outfile_type: str, - infile_structure: str, outfile_structure: str, averaging_method: str = 'mean'): + infile_structure: str, outfile_structure: str, **kwargs): self.infile = infile self.outfile = outfile self.infile_type = infile_type self.outfile_type = outfile_type self.infile_structure = infile_structure self.outfile_structure = outfile_structure - self.averaging_method = averaging_method self._temp_files = [] if not os.path.isfile(self.infile): @@ -111,6 +87,8 @@ def __init__(self, infile: str, outfile: str, infile_type: str, outfile_type: st ) self._converter = self.get_converter() + if not (self._converter.__class__ is BaseConvert): + self._converter.process_file(**kwargs) def get_converter(self): """ @@ -140,11 +118,9 @@ def get_converter(self): if self.infile_type == 'antennas_iq': if self.outfile_type == 'bfiq': - return ProcessAntennasIQ2Bfiq(self.infile, self.outfile, self.infile_structure, - self.outfile_structure) + return AntennasIQ2Bfiq(self.infile, self.outfile, self.infile_structure, self.outfile_structure) elif self.outfile_type == 'rawacf': - return ProcessAntennasIQ2Rawacf(self.infile, self.outfile, self.infile_structure, - self.outfile_structure, self.averaging_method) + return AntennasIQ2Rawacf(self.infile, self.outfile, self.infile_structure, self.outfile_structure) else: raise conversion_exceptions.ConversionUpstreamError( f'Conversion from {self.infile_type} to {self.outfile_type} is not supported. Only downstream ' @@ -153,8 +129,7 @@ def get_converter(self): ) elif self.infile_type == 'bfiq': if self.outfile_type == 'rawacf': - return ProcessBfiq2Rawacf(self.infile, self.outfile, self.infile_structure, self.outfile_structure, - self.averaging_method) + return Bfiq2Rawacf(self.infile, self.outfile, self.infile_structure, self.outfile_structure) else: raise conversion_exceptions.ConversionUpstreamError( f'Conversion from {self.infile_type} to {self.outfile_type} is not supported. Only downstream ' @@ -174,7 +149,7 @@ def main(): args = parser.parse_args() ConvertFile(args.infile, args.outfile, args.infile_type, args.outfile_type, args.infile_structure, - args.outfile_structure, averaging_method=args.averaging_method) + args.outfile_structure) if __name__ == "__main__": diff --git a/postprocessors/core/convert_base.py b/postprocessors/core/convert_base.py new file mode 100644 index 0000000..8ea5736 --- /dev/null +++ b/postprocessors/core/convert_base.py @@ -0,0 +1,306 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan +# Author: Remington Rohel +""" +This file contains base functionality for postprocessing of Borealis data files. +""" +import os +import traceback +from collections import OrderedDict +from typing import Union +import h5py +from functools import partial +from multiprocessing import get_context + +import postprocessors.core.restructure as rs +from postprocessors import conversion_exceptions + +try: + import cupy as xp +except ImportError: + import numpy as xp + cupy_available = False +else: + cupy_available = True + +import logging + +postprocessing_logger = logging.getLogger('borealis_postprocessing') + + +def processing_machine(idx: int, filename: str, record_keys: list, records_per_process: int, processing_fn, **kwargs): + """ + Helper function for processing a single record. It is defined here to facilitate multiprocessing. + + Parameters + ---------- + idx: int + Index into record_keys which tells processing_machine() which record to process + filename: str + HDF5 file with records to process. + record_keys: list + List of all top-level keys of the HDF5 file. + records_per_process: int + Number of records to process per call to this function. + processing_fn: callable + Function to call to process a record. + kwargs: dict + Key-word arguments to pass to processing_fn + + Returns + ------- + formatted_record, idx: properly-formatted processed record and the index which was processed. + """ + with h5py.File(filename, 'r') as hdf5_file: + record_dict = rs.read_group(hdf5_file[record_keys[idx]]) + record_list = [] # List of all 'extra' records to process + + # If processing multiple records at a time, get all the records ready + if records_per_process > 1: + for num in range(idx + 1, min(idx + records_per_process, len(record_keys))): + record_list.append(rs.read_group(hdf5_file[record_keys[num]])) + + processed_record = processing_fn(record_dict, extra_records=record_list, **kwargs) + + if processed_record is None: + return None, idx + else: + # Convert to numpy arrays for saving to file + formatted_record = rs.convert_to_numpy(processed_record) + return formatted_record, idx + + +class BaseConvert(object): + """ + Class for converting Borealis filetypes of all structures. This class abstracts and redirects + the file being converted to the correct class (ProcessAntennasIQ2Bfiq, ProcessAntennasIQ2Rawacf, + or ProcessBfiq2Rawacf). + + See Also + -------- + ProcessAntennasIQ2Bfiq + ProcessAntennasIQ2Rawacf + ProcessBfiq2Rawacf + ConvertFile + + Attributes + ---------- + infile: str + The filename of the input file containing SuperDARN data. + outfile: str + The file name of output file + infile_type: str + Type of data file. Types include: + 'antennas_iq' + 'bfiq' + 'rawacf' + outfile_type: str + Desired type of output data file. Same types as above. + infile_structure: str + The structure of the file. Structures include: + 'array' + 'site' + 'iqdat' (bfiq only) + 'dmap' (rawacf only) + All borealis files are either 'site' or 'array' structured. + outfile_structure: str + The desired structure of the output file. Same structures as + above. + """ + def __init__(self, infile: str, outfile: str, infile_type: str, outfile_type: str, infile_structure: str, + outfile_structure: str): + """ + Initializes the attributes of the class. + + Parameters + ---------- + infile: str + Path to the input file + outfile: str + Path to the output file + infile_type: str + Borealis file type of input file. Supported types are: + 'antennas_iq' + 'bfiq' + 'rawacf' + outfile_type: str + Borealis file type of output file. Supported types are same as for file_type. + infile_structure: str + Borealis file structure of input file. Supported structures are: + 'array' + 'site' + outfile_structure: str + Borealis file structure of output file. Supported structures are: + 'array' + 'site' + 'iqdat' (bfiq only) + 'dmap' (rawacf only) + """ + self.infile = infile + self.outfile = outfile + self.infile_type = infile_type + self.infile_structure = infile_structure + self.outfile_type = outfile_type + self.outfile_structure = outfile_structure + self.check_args() + + self.averaging_method = None + self._temp_files = [] + + def check_args(self): + + if self.infile_structure not in rs.FILE_STRUCTURE_MAPPING[self.infile_type]: + raise conversion_exceptions.ImproperFileStructureError( + f'Input file structure "{self.infile_structure}" is not compatible with input file type ' + f'"{self.infile_type}": Valid structures for {self.infile_type} are ' + f'{rs.FILE_STRUCTURE_MAPPING[self.infile_type]}' + ) + if self.outfile_structure not in rs.FILE_STRUCTURE_MAPPING[self.outfile_type]: + raise conversion_exceptions.ImproperFileStructureError( + f'Output file structure "{self.outfile_structure}" is not compatible with output file type ' + f'"{self.outfile_type}": Valid structures for {self.outfile_type} are ' + f'{rs.FILE_STRUCTURE_MAPPING[self.outfile_type]}' + ) + if self.infile_structure not in ['array', 'site']: + raise conversion_exceptions.ConversionUpstreamError( + f'Input file structure "{self.infile_structure}" cannot be reprocessed into any other format.' + ) + + def process_file(self, **kwargs): + """ + Applies appropriate downstream processing to convert between file types (for site-structured + files only). The processing chain is as follows: + 1. Restructure to site format + 2. Apply appropriate downstream processing by calling process_record() on each record + 3. Restructure to final format + 4. Remove all intermediate files created along the way + + Parameters + ---------- + **kwargs: dict + Supported kwargs include: + force: bool, if True will overwrite an existing output file + avg_num: int, how many records are grouped together for a single process_record() call + num_processes: int, how many CPU cores to distribute the job across + Other kwargs may be supported by child classes and will be passed through to the process_record() function. + """ + + if os.path.isfile(self.outfile) and not kwargs.get('force', False): + choice = input(f'Output file {self.outfile} already exists. Proceed anyway? Only records which don\'t ' + f'exist in output file will be processed. (y/n): ') + if choice[0] not in ['y', 'Y']: + return 0 + + try: + # Restructure to 'site' format if necessary + if self.infile_structure != 'site': + file_to_process = f'{self.infile}.site.tmp' + self._temp_files.append(file_to_process) + # Restructure file to site format for processing + postprocessing_logger.info(f'Restructuring file {self.infile} --> {file_to_process}') + rs.restructure(self.infile, file_to_process, self.infile_type, self.infile_structure, 'site') + else: + file_to_process = self.infile + + # Prepare to restructure after processing, if necessary + if self.outfile_structure != 'site': + processed_file = f'{self.outfile}.site.tmp' + self._temp_files.append(processed_file) + else: + processed_file = self.outfile + + postprocessing_logger.info(f'Converting file {file_to_process} --> {processed_file}') + + # First we want to check if any records have all been done, to lighten our workload + finished_records = set() + if os.path.isfile(processed_file) and not kwargs.get('force', False): + with h5py.File(processed_file, 'r') as f: + finished_records = set(f.keys()) + + # Load record names from file + with h5py.File(file_to_process, 'r') as infile: + all_records = sorted(list(infile.keys())) + + records_per_process = kwargs.get('avg_num', 1) # Records getting averaged together. + if not kwargs.get('force', False): # file may be partially processed, only process remaining records + final_records_remaining = sorted(list( + set(all_records[::records_per_process]).difference(finished_records))) + else: + final_records_remaining = all_records[::records_per_process] + + first_idx = all_records.index(final_records_remaining[0]) # first record to process + num_to_process = round(len(all_records) / records_per_process) + num_completed = first_idx + indices = range(first_idx, len(all_records), records_per_process) + + function_to_call = partial(processing_machine, + filename=file_to_process, record_keys=all_records, + records_per_process=records_per_process, + processing_fn=self.process_record, **kwargs) + + # Do the processing on each record + with h5py.File(processed_file, 'a') as outfile: + def append_to_file(rec): + """Convenience function to append to file""" + if rec is not None: + rs.write_records(outfile, {all_records[i]: rec}) + + def progress_bar(done_so_far, total): + """Convenience function to print a progress bar""" + completion_percentage = done_so_far / total + bar_width = 60 # arbitrary width + filled = int(bar_width * completion_percentage) + unfilled = bar_width - filled + print(f'\r[{"=" * filled}{" " * unfilled}] {completion_percentage * 100:.2f}%', flush=True, end='') + + if 'num_processes' in kwargs: # Use multiprocessing if specified + with get_context("spawn").Pool(kwargs.get('num_processes', 1)) as p: + for completed_record, i in p.imap(function_to_call, indices): + append_to_file(completed_record) + num_completed += 1 + progress_bar(num_completed, num_to_process) + else: # Default single-worker + for idx in indices: + completed_record, i = function_to_call(idx) + append_to_file(completed_record) + num_completed += 1 + progress_bar(num_completed, num_to_process) + print('\r', flush=True, end='') # Remove the progress bar + + # Restructure to final structure format, if necessary + if self.outfile_structure != 'site': + postprocessing_logger.info(f'Restructuring file {processed_file} --> {self.outfile}') + rs.restructure(processed_file, self.outfile, self.outfile_type, 'site', self.outfile_structure) + except (Exception,) as e: + postprocessing_logger.error(f'Could not process file {self.infile} -> {self.outfile}. Removing all newly' + f' generated files.') + postprocessing_logger.error(e) + postprocessing_logger.error(traceback.print_exc()) + finally: + self._remove_temp_files() + + def _remove_temp_files(self): + """ + Deletes all temporary files used in the processing chain. + """ + for filename in self._temp_files: + if os.path.exists(filename): + os.remove(filename) + + @classmethod + def process_record(cls, record: OrderedDict, **kwargs) -> OrderedDict: + """ + This method should be overwritten by child classes, and should contain the necessary + steps to process a record of input type to output type. + + Parameters + ---------- + record: OrderedDict + An hdf5 group containing one record of site-structured data + + Returns + ------- + record: OrderedDict + The same hdf5 group, but with the necessary modifications to conform to the standard + of data for self.final_type. + """ + return record diff --git a/postprocessors/core/restructure.py b/postprocessors/core/restructure.py new file mode 100644 index 0000000..3f910ca --- /dev/null +++ b/postprocessors/core/restructure.py @@ -0,0 +1,147 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan +# Author: Remington Rohel + +""" +This module provides some functions which are used in the processing of Borealis data files. +""" +import pydarnio +import numpy as np +import h5py + + +def read_group(group: h5py.Group): + """ + Reads a group from an HDF5 file into a dictionary. + + Parameters + ---------- + group: h5py.Group + Opened h5py group + + Returns + ------- + dict + Dictionary of {group_name: {}} where the inner dictionary is the datasets/attributes + of the hdf5 group. + """ + group_dict = {} + # Get the datasets (vector fields) + datasets = list(group.keys()) + for dset_name in datasets: + dset = group[dset_name] + if 'strtype' in dset.attrs: # string type, requires some handling + itemsize = dset.attrs['itemsize'] + data = dset[:].view(dtype=(np.unicode_, itemsize)) + else: + data = dset[:] # non-string, can simply load + group_dict[dset_name] = data + + # Get the attributes (scalar fields) + attribute_dict = {} + for k, v in group.attrs.items(): + if k in ['CLASS', 'TITLE', 'VERSION', 'DEEPDISH_IO_VERSION', 'PYTABLES_FORMAT_VERSION']: + continue + elif isinstance(v, np.bytes_): + if v.itemsize == 0: + attribute_dict[k] = '' + else: + attribute_dict[k] = v.tobytes().decode('utf-8') + elif isinstance(v, h5py.Empty): + dtype = v.dtype.type + data = dtype() + if isinstance(data, bytes): + data = data.decode('utf-8') + attribute_dict[k] = data + else: + attribute_dict[k] = v + group_dict.update(attribute_dict) + + return group_dict + + +def write_records(hdf5_file: h5py.File, records: dict): + """ + Write the record to file. + + Parameters + ---------- + hdf5_file: h5py.File + HDF5 file to write records to. + records: dict + Dictionary containing fields to write to file. + """ + for group_name, group_dict in records.items(): + group = hdf5_file.create_group(str(group_name)) + for k, v in group_dict.items(): + if isinstance(v, str): + group.attrs[k] = np.bytes_(v) + elif isinstance(v, np.ndarray): + if v.dtype.type == np.str_: + dset = group.create_dataset(k, data=v.view(dtype=np.uint8)) + dset.attrs['strtype'] = b'unicode' + dset.attrs['itemsize'] = v.dtype.itemsize // 4 # every character is 4 bytes + else: + group.create_dataset(k, data=v) + else: + group.attrs[k] = v + + +def restructure(infile_name, outfile_name, infile_type, infile_structure, outfile_structure): + """ + This method restructures filename of structure "file_structure" into "final_structure". + + Parameters + ---------- + infile_name: str + Name of the original file. + outfile_name: str + Name of the restructured file. + infile_type: str + Borealis file type of the files. + infile_structure: str + The current write structure of the file. One of 'array' or 'site'. + outfile_structure: str + The desired write structure of the file. One of 'array', 'site', 'iqdat', or 'dmap'. + """ + # dmap and iqdat are not borealis formats, so they are handled specially + if outfile_structure == 'dmap' or outfile_structure == 'iqdat': + pydarnio.BorealisConvert(infile_name, infile_type, outfile_name, + borealis_file_structure=infile_structure) + return + + pydarnio.BorealisRestructure(infile_name, outfile_name, infile_type, outfile_structure) + + +def convert_to_numpy(data: dict): + """Converts lists stored in dict into numpy array. Recursive. + Args: + data (Python dictionary): Dictionary with lists to convert to numpy arrays. + """ + for k, v in data.items(): + if isinstance(v, dict): + convert_to_numpy(v) + elif isinstance(v, list): + data[k] = np.array(v) + else: + continue + + return data + + +# Dictionary mapping the accepted borealis file types to the borealis file types that they can be processed into. +# The dictionary keys are the valid input file types, and their values are lists of file types which they can be +# processed into. +FILE_TYPE_MAPPING = { + 'antennas_iq': ['antennas_iq', 'bfiq', 'rawacf'], + 'bfiq': ['bfiq', 'rawacf'], + 'rawacf': ['rawacf'] +} + +# Dictionary mapping the accepted borealis file types to the borealis file structures that they can be formatted as. +# The dictionary keys are the valid input file types, and their values are lists of file structures which they can be +# formatted as. +FILE_STRUCTURE_MAPPING = { + 'antennas_iq': ['site', 'array'], + 'bfiq': ['site', 'array', 'iqdat'], + 'rawacf': ['site', 'array', 'dmap'] +} diff --git a/data_processing/__init__.py b/postprocessors/exceptions/__init__.py similarity index 100% rename from data_processing/__init__.py rename to postprocessors/exceptions/__init__.py diff --git a/exceptions/conversion_exceptions.py b/postprocessors/exceptions/conversion_exceptions.py similarity index 100% rename from exceptions/conversion_exceptions.py rename to postprocessors/exceptions/conversion_exceptions.py diff --git a/postprocessors/exceptions/processing_exceptions.py b/postprocessors/exceptions/processing_exceptions.py new file mode 100644 index 0000000..5d9d2d4 --- /dev/null +++ b/postprocessors/exceptions/processing_exceptions.py @@ -0,0 +1,36 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan +# Author: Remington Rohel +""" +This files contains the exceptions generated when an impossible +conversion is attempted. +""" + +import logging +postprocessing_logger = logging.getLogger('borealis_postprocessing') + + +class FileCreationError(Exception): + """ + Raised when a file cannot be created. + + Parameters + ---------- + error_str: str + explanation for why the error was raised. + + Attributes + ---------- + message: str + The message to display with the error + + See Also + -------- + antennas_iq_to_bfiq.py + bfiq_to_rawacf.py + """ + + def __init__(self, error_str: str): + self.message = "File cannot be made: {error_str}"\ + "".format(error_str=error_str) + postprocessing_logger.error(self.message) + Exception.__init__(self, self.message) diff --git a/data_processing/sandbox/__init__.py b/postprocessors/sandbox/__init__.py similarity index 100% rename from data_processing/sandbox/__init__.py rename to postprocessors/sandbox/__init__.py diff --git a/postprocessors/sandbox/binomial_beams.py b/postprocessors/sandbox/binomial_beams.py new file mode 100644 index 0000000..480e729 --- /dev/null +++ b/postprocessors/sandbox/binomial_beams.py @@ -0,0 +1,57 @@ +# Copyright 2023 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for converting antennas_iq files from widebeam experiments +to rawacf files, using a Binomial window in amplitude for beamforming to reduce receiver sidelobes. +""" +from postprocessors import AntennasIQ2Rawacf + + +class BinomialWindowBeamforming(AntennasIQ2Rawacf): + """ + Class for conversion of Borealis antennas_iq files into rawacf files for beam-broadening experiments. This class + inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. The beams + are formed using a Binomial window and standard beamforming, to remove sidelobes. + + See Also + -------- + ConvertFile + BaseConvert + HammingWindowBeamforming + + Attributes + ---------- + infile: str + The filename of the input antennas_iq file. + outfile: str + The file name of output file + infile_structure: str + The write structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + window = [0.0001554001554001554, 0.002331002331002331, 0.016317016317016316, 0.0707070707070707, + 0.21212121212121213, 0.4666666666666667, 0.7777777777777778, 1.0, + 1.0, 0.7777777777777778, 0.4666666666666667, 0.21212121212121213, + 0.0707070707070707, 0.016317016317016316, 0.002331002331002331, 0.0001554001554001554] + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, **kwargs): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, infile_structure, outfile_structure) + + self.process_file(**kwargs) diff --git a/postprocessors/sandbox/bistatic_processing.py b/postprocessors/sandbox/bistatic_processing.py new file mode 100644 index 0000000..3afdb39 --- /dev/null +++ b/postprocessors/sandbox/bistatic_processing.py @@ -0,0 +1,143 @@ +# Copyright 2022 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for processing bistatic_test antennas_iq files to rawacf. +""" +from collections import OrderedDict +from typing import Union +import h5py +import numpy as np + +from postprocessors import BaseConvert, AntennasIQ2Rawacf + + +class BistaticProcessing(BaseConvert): + """ + Class for processing bistatic_test antennas_iq files to rawacf. To properly use this class, you first must + extract the timestamps from the transmitting radar using the ExtractTimestamps class. These are cross-referenced + against the timestamps in the receiving radar file (infile for this class), so that only sequences which occurred + at the same time for both radars are processed. + + This class inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. + + See Also + -------- + ConvertFile + BaseConvert + ProcessAntennasIQ2Rawacf + + Attributes + ---------- + infile: str + The filename of the input file. + outfile: str + The file name of output file + infile_structure: str + The write structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, timestamps_file: str): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + timestamps_file: str + Path to file containing transmitting radar's sequence timestamps + """ + super().__init__(infile, outfile, 'antennas_iq', 'rawacf', infile_structure, outfile_structure) + + with h5py.File(timestamps_file, 'r') as tstamp_file: + keys = sorted(list(tstamp_file.keys())) + + timestamps = np.concatenate([tstamp_file[k]['data']['sqn_timestamps'][:] for k in keys]) + timestamps = np.around(timestamps, decimals=6) # round to nearest microsecond + + self.process_file(timestamps=timestamps) + + @staticmethod + def process_record(record: OrderedDict, averaging_method: Union[None, str], **kwargs) -> Union[OrderedDict, None]: + """ + Takes a record from a Borealis file and only processes the sequences which were transmitted by + the transmitting radar site. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing antennas_iq data and metadata + averaging_method: Union[None, str] + Method to use for averaging correlations across sequences. Acceptable methods are 'median' and 'mean' + **kwargs: dict + timestamps: ndarray + 1D array of timestamps in seconds past epoch + + Returns + ------- + record: OrderedDict + hdf5 record + """ + tx_timestamps = kwargs['timestamps'] + + rx_tstamps = np.around(record['sqn_timestamps'], decimals=6) # Round to nearest microsecond + + # Get the tx timestamps that occur during this record + tx_after_start = np.argwhere(rx_tstamps[0] <= tx_timestamps) + tx_before_end = np.argwhere(tx_timestamps <= rx_tstamps[-1]) + if tx_after_start.size == 0 or tx_before_end.size == 0: + # No overlap in times, so throw out this record + return None + else: + tx_start = np.min(tx_after_start) + tx_end = np.max(tx_before_end) + tx_times = tx_timestamps[tx_start:tx_end + 1] # Only the times that overlap, including the end + + # Keep all indices from rx_tstamps that have a corresponding tx timestamp + keep_indices = [] + j = 0 + for i in range(len(rx_tstamps)): + if j == tx_times.size: # No more tx sequences + break + if rx_tstamps[i] == tx_times[j]: # Both radars synced and operated during this time, add sequence + keep_indices.append(i) + j += 1 + elif abs(rx_tstamps[i] - tx_times[j]) < 1e-3: # Both radars operated, but not synced up, so skip it + j += 1 + elif rx_tstamps[i] < tx_times[j]: # tx radar missed a sequence, skip this rx sequence + pass + else: # rx radar missed a sequence so is ahead, gotta catch tx radar back up + while rx_tstamps[i] - tx_times[j] > 1e-3 and j < tx_times.size - 1: + j += 1 + if rx_tstamps[i] == tx_times[j]: + # Still have to check if they are within a microsecond of each other + keep_indices.append(i) + j += 1 + + if len(keep_indices) == 0: # No overlapping timestamps, throw out this record + return None + + # Extract the good data + data_dimensions = record['data_dimensions'] + data = record['data'].reshape(data_dimensions) + record['data'] = data[:, keep_indices, :].flatten() # [num_antennas, num_sequences, num_samps] + + # Update the metadata + data_dimensions[1] = np.uint32(len(keep_indices)) + record['data_dimensions'] = data_dimensions + record['num_sequences'] = np.int64(len(keep_indices)) + record['sqn_timestamps'] = record['sqn_timestamps'][keep_indices] + + record = AntennasIQ2Rawacf.process_record(record, averaging_method) + + return record diff --git a/postprocessors/sandbox/extract_beam.py b/postprocessors/sandbox/extract_beam.py new file mode 100644 index 0000000..ab44409 --- /dev/null +++ b/postprocessors/sandbox/extract_beam.py @@ -0,0 +1,109 @@ +# Copyright 2022 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for extracting sequence timestamps from any file type. +""" +from collections import OrderedDict +from typing import Union + +from postprocessors import BaseConvert + + +class ExtractBeamsFromRawacf(BaseConvert): + """ + Class for extraction of beams from rawacf files. + This class inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. + + See Also + -------- + ConvertFile + BaseConvert + ProcessAntennasIQ2Bfiq + ProcessBfiq2Rawacf + ProcessAntennasIQ2Rawacf + + Attributes + ---------- + infile: str + The filename of the input file. + outfile: str + The file name of output file + infile_structure: str + The write structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, **kwargs): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, 'rawacf', 'rawacf', infile_structure, outfile_structure) + + beam_str = kwargs.get('beams', '') + if beam_str == '': + raise ValueError('No beams defined') + + beam_list = [] + for beam_chunk in beam_str.split(','): + if '-' in beam_chunk: + first_beam, last_beam = beam_chunk.split('-') + beam_list.extend([i for i in range(int(first_beam), int(last_beam) + 1)]) # include the endpoints + else: + beam_list.append(int(beam_chunk)) + + self.process_file(beams=beam_list) + + @staticmethod + def process_record(record: OrderedDict, **kwargs) -> Union[OrderedDict, None]: + """ + Takes a record from a Borealis rawacf file and extracts only specific beams. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing rawacf data and metadata. + kwargs: dict + beams: list of integers corresponding to beam numbers to extract. + + Returns + ------- + record: OrderedDict + hdf5 record, containing only the sequence timestamps. + """ + beams = kwargs.get('beams', []) + record_beams = record['beam_nums'] + + # Find the matching beams in the record + matching_indices = [i for i in range(len(record_beams)) if record_beams[i] in beams] + + if len(matching_indices) == 0: + return None + + record_dims = record['correlation_dimensions'] + for field in ['main_acfs', 'intf_acfs', 'xcfs']: + data = record[field] + data = data.reshape(record_dims) + # data is [num_beams, num_ranges, num_lags] + record[field] = data[matching_indices].flatten() + + record_dims[0] = len(matching_indices) + + record['correlation_dimensions'] = record_dims + record['beam_azms'] = record['beam_azms'][matching_indices] + record['beam_nums'] = record['beam_nums'][matching_indices] + + return record diff --git a/postprocessors/sandbox/extract_timestamps.py b/postprocessors/sandbox/extract_timestamps.py new file mode 100644 index 0000000..8579681 --- /dev/null +++ b/postprocessors/sandbox/extract_timestamps.py @@ -0,0 +1,83 @@ +# Copyright 2022 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for extracting sequence timestamps from any file type. +""" +from collections import OrderedDict +from typing import Union + +from postprocessors import BaseConvert + + +class ExtractTimestamps(BaseConvert): + """ + Class for extraction of sequence timestamps for bistatic experiments. + This class inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. + + See Also + -------- + ConvertFile + BaseConvert + ProcessAntennasIQ2Bfiq + ProcessBfiq2Rawacf + ProcessAntennasIQ2Rawacf + + Attributes + ---------- + infile: str + The filename of the input file. + outfile: str + The file name of output file + infile_structure: str + The write structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + + def __init__(self, infile: str, outfile: str, infile_type: str, outfile_type: str, + infile_structure: str, outfile_structure: str): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_type: str + Borealis filetype of input file. 'antennas_iq', 'bfiq', or 'rawacf'. + outfile_type: str + Borealis filetype of output file. 'antennas_iq', 'bfiq', or 'rawacf'. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, infile_type, outfile_type, infile_structure, outfile_structure) + + self.process_file() + + @staticmethod + def process_record(record: OrderedDict, averaging_method: Union[None, str], **kwargs) -> OrderedDict: + """ + Takes a record from a Borealis file and extract the sequence timestamps only. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing antennas_iq data and metadata + averaging_method: Union[None, str] + Method to use for averaging correlations across sequences. Acceptable methods are 'median' and 'mean' + + Returns + ------- + record: OrderedDict + hdf5 record, containing only the sequence timestamps. + """ + new_record = OrderedDict() + new_record['sqn_timestamps'] = record['sqn_timestamps'] + + return new_record diff --git a/postprocessors/sandbox/hamming_beams.py b/postprocessors/sandbox/hamming_beams.py new file mode 100644 index 0000000..dd06132 --- /dev/null +++ b/postprocessors/sandbox/hamming_beams.py @@ -0,0 +1,58 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for converting antennas_iq files from widebeam experiments +to rawacf files, using a Hamming window in amplitude for beamforming to reduce receiver sidelobes. +""" +from postprocessors import AntennasIQ2Rawacf + + +class HammingWindowBeamforming(AntennasIQ2Rawacf): + """ + Class for conversion of Borealis antennas_iq files into rawacf files for beam-broadening experiments. This class + inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. The beams + are formed using a Hamming window and standard beamforming, to keep the largest sidelobe down 40 dB below the + main lobe. + + See Also + -------- + ConvertFile + BaseConvert + ProcessAntennasIQ2Bfiq + ProcessBfiq2Rawacf + ProcessAntennasIQ2Rawacf + + Attributes + ---------- + infile: str + The filename of the input antennas_iq file. + outfile: str + The file name of output file + infile_structure: str + The structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + window = [0.08081232549588463, 0.12098514265395757, 0.23455777475180511, 0.4018918165398586, + 0.594054435182454, 0.7778186328978896, 0.9214100134552521, 1.0, + 1.0, 0.9214100134552521, 0.7778186328978896, 0.594054435182454, + 0.4018918165398586, 0.23455777475180511, 0.12098514265395757, 0.08081232549588463] + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, **kwargs): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, infile_structure, outfile_structure) diff --git a/postprocessors/sandbox/hamming_corrected.py b/postprocessors/sandbox/hamming_corrected.py new file mode 100644 index 0000000..f49ac4a --- /dev/null +++ b/postprocessors/sandbox/hamming_corrected.py @@ -0,0 +1,152 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for converting antennas_iq files from widebeam experiments +to rawacf files, using a Hamming window in amplitude for beamforming to reduce receiver sidelobes. +Receiver beam directions are adjusted to correct for azimuthal sensitivity variations. +""" +import copy +from collections import OrderedDict +import numpy as np + +from postprocessors.sandbox.hamming_beams import HammingWindowBeamforming + + +class HammingBeamformingCorrected(HammingWindowBeamforming): + """ + Class for conversion of Borealis antennas_iq files into rawacf files for beam-broadening experiments. This class + inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. The beams + are formed using a Hamming window and standard beamforming, to keep the largest sidelobe down 40 dB below the + main lobe. + + See Also + -------- + ConvertFile + BaseConvert + ProcessAntennasIQ2Bfiq + ProcessBfiq2Rawacf + ProcessAntennasIQ2Rawacf + + Attributes + ---------- + infile: str + The filename of the input antennas_iq file. + outfile: str + The file name of output file + infile_structure: str + The structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + window = [0.08081232549588463, 0.12098514265395757, 0.23455777475180511, 0.4018918165398586, + 0.594054435182454, 0.7778186328978896, 0.9214100134552521, 1.0, + 1.0, 0.9214100134552521, 0.7778186328978896, 0.594054435182454, + 0.4018918165398586, 0.23455777475180511, 0.12098514265395757, 0.08081232549588463] + + acf_directions = { + 10400: [-26.9, -22.66, -19.22, -14.38, -11.34, -7.7, -3.76, -1.52, + 1.52, 3.76, 7.8, 11.34, 14.38, 19.22, 22.66, 26.9], + 10500: [-27., -22.56, -19.22, -14.48, -11.34, -7.8, -3.76, -1.52, + 1.52, 3.76, 7.8, 11.34, 14.48, 19.22, 22.56, 27.], + 10600: [-27.1, -22.46, -19.22, -14.48, -11.32, -7.8, -3.76, -1.46, + 1.46, 3.76, 7.8, 11.32, 14.48, 19.22, 22.46, 27.1], + 10700: [-27.2, -22.46, -19.32, -14.58, -11.27, -7.9, -3.76, -1.46, + 1.46, 3.76, 7.9, 11.27, 14.58, 19.32, 22.46, 27.3], + 10800: [-27.3, -22.46, -19.32, -14.68, -11.24, -7.9, -3.76, -1.46, + 1.46, 3.76, 7.9, 11.24, 14.68, 19.32, 22.46, 27.4], + 10900: [-27.4, -22.36, -19.32, -14.78, -11.24, -7.9, -3.76, -1.46, + 1.46, 3.76, 8., 11.24, 14.78, 19.32, 22.36, 27.5], + 12200: [-26.6, -21.66, -17.42, -14.84, -11.24, -8.1, -4.86, -1.82, + 1.82, 4.86, 8.1, 11.24, 14.84, 17.42, 21.66, 26.7], + 12300: [-26.7, -21.76, -17.52, -14.78, -11.24, -8.1, -4.86, -1.82, + 1.82, 4.86, 8.1, 11.24, 14.78, 17.52, 21.76, 26.8], + 12500: [-26.9, -21.86, -17.52, -14.74, -11.34, -7.9, -4.86, -1.92, + 1.92, 4.86, 7.9, 11.34, 14.74, 17.52, 21.86, 27.], + 13000: [-27.3, -22.06, -17.52, -14.64, -11.24, -7.7, -4.96, -2.12, + 2.12, 4.96, 7.7, 11.24, 14.64, 17.52, 22.06, 27.5], + 13100: [-27.7, -22.06, -17.62, -14.54, -11.44, -7.6, -4.93, -2.12, + 2.12, 4.93, 7.6, 11.44, 14.54, 17.62, 22.06, 27.8], + 13200: [-27.7, -22.16, -17.72, -14.54, -11.34, -7.6, -4.96, -2.12, + 2.12, 4.96, 7.6, 11.34, 14.54, 17.72, 22.16, 27.8], + } + xcf_directions = { + 10400: [-28.8, -23.96, -19.92, -14.68, -11.24, -7.4, -3.48, -1.36, + 1.36, 3.48, 7.5, 11.24, 14.68, 19.92, 23.96, 28.8], + 10500: [-28.8, -23.86, -20.02, -14.78, -11.24, -7.5, -3.53, -1.41, + 1.41, 3.53, 7.5, 11.24, 14.78, 20.02, 23.86, 29.], + 10600: [-29., -23.76, -20.02, -14.78, -11.24, -7.5, -3.48, -1.32, + 1.32, 3.48, 7.6, 11.24, 14.78, 20.02, 23.76, 29.], + 10700: [-29.1, -23.76, -20.12, -14.88, -11.24, -7.6, -3.53, -1.32, + 1.32, 3.53, 7.6, 11.24, 14.88, 20.12, 23.76, 29.2], + 10800: [-29.3, -23.76, -20.12, -14.98, -11.24, -7.6, -3.53, -1.32, + 1.32, 3.53, 7.7, 11.24, 14.98, 20.12, 23.76, 29.4], + 10900: [-29.3, -23.66, -20.12, -15.08, -11.24, -7.7, -3.56, -1.32, + 1.32, 3.56, 7.7, 11.24, 15.08, 20.12, 23.66, 29.4], + 12200: [-28.4, -22.26, -17.62, -14.78, -11.24, -8.15, -4.96, -1.82, + 1.82, 4.96, 8.15, 11.24, 14.78, 17.62, 22.26, 28.5], + 12300: [-28.5, -22.46, -17.62, -14.78, -11.24, -8.1, -4.96, -1.82, + 1.82, 4.96, 8.1, 11.24, 14.78, 17.62, 22.46, 28.6], + 12500: [-28.7, -22.56, -17.62, -14.69, -11.24, -7.9, -4.88, -1.92, + 1.92, 4.88, 7.9, 11.24, 14.69, 17.62, 22.56, 28.8], + 13000: [-29.3, -22.86, -17.72, -14.58, -11.14, -7.6, -4.96, -2.12, + 2.12, 4.96, 7.6, 11.14, 14.58, 17.72, 22.86, 29.4], + 13100: [-29.6, -22.96, -17.82, -14.48, -11.34, -7.55, -4.93, -2.12, + 2.12, 4.93, 7.55, 11.34, 14.48, 17.82, 22.96, 29.7], + 13200: [-29.6, -23.06, -17.92, -14.48, -11.24, -7.6, -4.96, -2.12, + 2.12, 4.96, 7.6, 11.24, 14.48, 17.92, 23.06, 29.7], + } + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, **kwargs): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, infile_structure, outfile_structure) + + @classmethod + def process_record(cls, record: OrderedDict, **kwargs) -> OrderedDict: + """ + Overwrites the beam directions before beamforming then passes off to parent class processing. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing antennas_iq data and metadata + + Returns + ------- + record: OrderedDict + hdf5 record, with new fields required by bfiq data format + """ + acf_record = copy.deepcopy(record) + beam_nums = record['beam_nums'] + freq_khz = record['freq'] + old_beam_azms = record['beam_azms'] + acf_beam_azms = np.array([cls.acf_directions[freq_khz][i] for i in beam_nums]) + xcf_beam_azms = np.array([cls.xcf_directions[freq_khz][i] for i in beam_nums]) + acf_record['beam_azms'] = acf_beam_azms + record['beam_azms'] = xcf_beam_azms + + # Now do the processing with the new beam directions + acf_record = super().process_record(acf_record) + record = super().process_record(record) + + record['main_acfs'] = acf_record['main_acfs'] + + # The direction of sensitivity is still the same, so revert back to original beam directions. + record['beam_azms'] = old_beam_azms + + return record + diff --git a/postprocessors/sandbox/hamming_corrected_8_antennas.py b/postprocessors/sandbox/hamming_corrected_8_antennas.py new file mode 100644 index 0000000..3ebfcb7 --- /dev/null +++ b/postprocessors/sandbox/hamming_corrected_8_antennas.py @@ -0,0 +1,65 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for converting antennas_iq files from widebeam experiments +to rawacf files, using a Hamming window in amplitude for beamforming to reduce receiver sidelobes. +Receiver beam directions are adjusted to correct for azimuthal sensitivity variations. +""" + +from postprocessors.sandbox.hamming_corrected import HammingBeamformingCorrected + + +class Hamming8Antennas(HammingBeamformingCorrected): + """ + Class for conversion of Borealis antennas_iq files into rawacf files for beam-broadening experiments. This class + inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. The beams + are formed using a Hamming window and standard beamforming, to keep the largest sidelobe down 40 dB below the + main lobe. + + See Also + -------- + ConvertFile + BaseConvert + ProcessAntennasIQ2Bfiq + ProcessBfiq2Rawacf + ProcessAntennasIQ2Rawacf + + Attributes + ---------- + infile: str + The filename of the input antennas_iq file. + outfile: str + The file name of output file + infile_structure: str + The structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + + acf_directions = { + 10800: [-28.2, -23.46, -18.32, -13.48, -10.14, -8.2, -6.08, -2.42, + 2.42, 6.08, 8.2, 10.14, 13.48, 18.32, 23.46, 28.2], + } + xcf_directions = { + 10800: [-30., -24.96, -18.72, -13.18, -9.97, -8.2, -6.26, -2.62, + 2.62, 6.26, 8.2, 9.97, 13.18, 18.72, 24.96, 30.], + } + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, **kwargs): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, infile_structure, outfile_structure) diff --git a/data_processing/antennas_iq_to_rawacf.py b/postprocessors/sandbox/process_impt.py similarity index 60% rename from data_processing/antennas_iq_to_rawacf.py rename to postprocessors/sandbox/process_impt.py index 6b2e950..28fae58 100644 --- a/data_processing/antennas_iq_to_rawacf.py +++ b/postprocessors/sandbox/process_impt.py @@ -1,18 +1,17 @@ # Copyright 2021 SuperDARN Canada, University of Saskatchewan # Author: Remington Rohel """ -This file contains functions for converting antennas_iq files -to rawacf files. +This file contains functions for adding a lag table and processing bfiq data into rawacf data. """ from collections import OrderedDict from typing import Union -from data_processing.convert_base import BaseConvert -from data_processing.antennas_iq_to_bfiq import ProcessAntennasIQ2Bfiq -from data_processing.bfiq_to_rawacf import ProcessBfiq2Rawacf +import numpy as np +from postprocessors import BaseConvert, Bfiq2Rawacf -class ProcessAntennasIQ2Rawacf(BaseConvert): + +class ProcessIMPT(BaseConvert): """ Class for conversion of Borealis antennas_iq files into rawacf files. This class inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. @@ -21,7 +20,6 @@ class ProcessAntennasIQ2Rawacf(BaseConvert): -------- ConvertFile BaseConvert - ProcessAntennasIQ2Bfiq ProcessBfiq2Rawacf Attributes @@ -56,7 +54,7 @@ def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_str averaging_method: str Method for averaging correlations across sequences. Either 'median' or 'mean'. """ - super().__init__(infile, outfile, 'antennas_iq', 'rawacf', infile_structure, outfile_structure) + super().__init__(infile, outfile, 'bfiq', 'rawacf', infile_structure, outfile_structure) self.averaging_method = averaging_method self.process_file() @@ -64,7 +62,7 @@ def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_str @staticmethod def process_record(record: OrderedDict, averaging_method: Union[None, str], **kwargs) -> OrderedDict: """ - Takes a record from an antennas_iq file process into a rawacf record. + Takes a record from an ImptTest bfiq file process into a rawacf record. Parameters ---------- @@ -78,7 +76,31 @@ def process_record(record: OrderedDict, averaging_method: Union[None, str], **kw record: OrderedDict hdf5 record, with new fields required by rawacf data format """ - record = ProcessAntennasIQ2Bfiq.process_record(record, averaging_method) - record = ProcessBfiq2Rawacf.process_record(record, averaging_method) + STD_8P_LAG_TABLE = [[0, 0], + [42, 43], + [22, 24], + [24, 27], + [27, 31], + [22, 27], + [24, 31], + [14, 22], + [22, 31], + [14, 24], + [31, 42], + [31, 43], + [14, 27], + [0, 14], + [27, 42], + [27, 43], + [14, 31], + [24, 42], + [24, 43], + [22, 42], + [22, 43], + [0, 22], + [0, 24], + [43, 43]] + record['lags'] = np.array(STD_8P_LAG_TABLE, dtype=np.uint32) + record = Bfiq2Rawacf.process_record(record, averaging_method) return record diff --git a/postprocessors/sandbox/rawacf_lag0.py b/postprocessors/sandbox/rawacf_lag0.py new file mode 100644 index 0000000..3f326e4 --- /dev/null +++ b/postprocessors/sandbox/rawacf_lag0.py @@ -0,0 +1,108 @@ +# Copyright 2021 SuperDARN Canada, University of Saskatchewan +# Author: Remington Rohel +""" +This file contains functions for replacing cluttered lag0 ranges with +data from alternate lag0. +""" +from collections import OrderedDict +from typing import Union +import numpy as np + +from postprocessors import BaseConvert + + +class ReprocessRawacfLag0(BaseConvert): + """ + Class for reprocessing of Borealis rawacf hdf5 files to replace lag0 data from ranges which contain + second-pulse clutter with alternate lag0 data. + + See Also + -------- + ConvertFile + BaseConvert + ProcessAntennasIQ2Bfiq + ProcessBfiq2Rawacf + + Attributes + ---------- + infile: str + The filename of the input antennas_iq file. + outfile: str + The file name of output file + infile_structure: str + The write structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + """ + super().__init__(infile, outfile, 'rawacf', 'rawacf', infile_structure, outfile_structure) + + self.process_file() + + @staticmethod + def process_record(record: OrderedDict, averaging_method: Union[None, str]) -> OrderedDict: + """ + Takes a record from an antennas_iq file process into a rawacf record. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing antennas_iq data and metadata + averaging_method: Union[None, str] + Method to use for averaging correlations across sequences. Acceptable methods are 'median' and 'mean' + + Returns + ------- + record: OrderedDict + hdf5 record, with new fields required by rawacf data format + """ + # Get the correlation data + main_acfs = record['main_acfs'] + intf_acfs = record['intf_acfs'] + xcfs = record['xcfs'] + + correlation_descriptors = record['correlation_descriptors'] + correlation_dimensions = record['correlation_dimensions'] + + main_acfs = main_acfs.reshape(correlation_dimensions) + intf_acfs = intf_acfs.reshape(correlation_dimensions) + xcfs = xcfs.reshape(correlation_dimensions) + + # Convert tau spacing into units of samples + tau_in_samples = record['tau_spacing'] * 1e-6 * record['rx_sample_rate'] + + # First range offset in units of samples + sample_off = record['first_range_rtt'] * 1e-6 * record['rx_sample_rate'] + sample_off = np.int32(sample_off) + + # Start of second pulse in units of samples + second_pulse_sample_num = np.int32(tau_in_samples) * record['pulses'][1] - sample_off - 1 + + # Replace all ranges which are contaminated by the second pulse for lag 0 + # with the data from those ranges after the final pulse. + main_acfs[:, second_pulse_sample_num:, 0] = main_acfs[:, second_pulse_sample_num:, -1] + intf_acfs[:, second_pulse_sample_num:, 0] = intf_acfs[:, second_pulse_sample_num:, -1] + xcfs[:, second_pulse_sample_num:, 0] = xcfs[:, second_pulse_sample_num:, -1] + + record['main_acfs'] = main_acfs + record['intf_acfs'] = intf_acfs + record['xcfs'] = xcfs + + return record diff --git a/postprocessors/sandbox/rawacf_record_averaging.py b/postprocessors/sandbox/rawacf_record_averaging.py new file mode 100644 index 0000000..1ad8be0 --- /dev/null +++ b/postprocessors/sandbox/rawacf_record_averaging.py @@ -0,0 +1,117 @@ +# Copyright 2022 SuperDARN Canada, University of Saskatchewan + +""" +This file contains functions for averaging multiple records of a rawacf file. +""" +from collections import OrderedDict +from typing import Union +import numpy as np + +from postprocessors import BaseConvert + + +class AverageMultipleRawacfRecords(BaseConvert): + """ + Class for averaging multiple rawacf records together. This class inherits from BaseConvert, which handles all + functionality generic to postprocessing borealis files. + + See Also + -------- + ConvertFile + BaseConvert + ProcessBfiq2Rawacf + ProcessAntennasIQ2Rawacf + + Attributes + ---------- + infile: str + The filename of the input rawacf file. + outfile: str + The file name of output file + infile_structure: str + The write structure of the file. Structures include: + 'array' + 'site' + outfile_structure: str + The desired structure of the output file. Same structures as above, plus 'dmap'. + """ + + def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_structure: str, + num_records: int = 2): + """ + Initialize the attributes of the class. + + Parameters + ---------- + infile: str + Path to input file. + outfile: str + Path to output file. + infile_structure: str + Borealis structure of input file. Either 'array' or 'site'. + outfile_structure: str + Borealis structure of output file. Either 'array', 'site', or 'dmap'. + num_records: int + Number of records to average together. Default 2. + """ + super().__init__(infile, outfile, 'rawacf', 'rawacf', infile_structure, outfile_structure) + + self._num_records = num_records + + self.process_file(avg_num=self._num_records) + + @staticmethod + def process_record(record: OrderedDict, averaging_method: Union[None, str] = 'mean', **kwargs) -> OrderedDict: + """ + Takes a record from a rawacf file and averages a specified number of adjacent records together. + + Parameters + ---------- + record: OrderedDict + hdf5 record containing antennas_iq data and metadata + averaging_method: Union[None, str] + Method to use for averaging correlations across sequences. For this class, only 'mean' averaging is + supported, as the median cannot be taken across multiple records after rawacf files have been made. + kwargs: + Supported key: 'extra_records' + 'extra_records' should be a list of OrderedDicts, which are the records to average. + Returns + ------- + record: OrderedDict + hdf5 record + """ + if 'extra_records' not in kwargs: + print("No extra records given.") + return record + + total_sequences = record['num_sequences'] + sqn_timestamps = list(record['sqn_timestamps']) + int_time = record['int_time'] + noise_at_freq = list(record['noise_at_freq']) + main_acfs = record['main_acfs'] * total_sequences + intf_acfs = record['intf_acfs'] * total_sequences + xcfs = record['xcfs'] * total_sequences + + for rec in kwargs['extra_records']: + num_sequences = rec['num_sequences'] + total_sequences += num_sequences + int_time += rec['int_time'] + noise_at_freq.extend(rec['noise_at_freq']) + sqn_timestamps.extend(list(rec['sqn_timestamps'])) + main_acfs += rec['main_acfs'] * num_sequences + intf_acfs += rec['intf_acfs'] * num_sequences + xcfs += rec['xcfs'] * num_sequences + + main_acfs /= total_sequences + intf_acfs /= total_sequences + xcfs /= total_sequences + + record['main_acfs'] = np.array(main_acfs, dtype=np.complex64) + record['intf_acfs'] = np.array(intf_acfs, dtype=np.complex64) + record['xcfs'] = np.array(xcfs, dtype=np.complex64) + record['int_time'] = int_time + record['num_sequences'] = total_sequences + record['sqn_timestamps'] = sqn_timestamps + record['noise_at_freq'] = noise_at_freq + + return record diff --git a/data_processing/sandbox/widebeam_antennas_iq_to_rawacf.py b/postprocessors/sandbox/widebeam_antennas_iq_to_bfiq.py similarity index 78% rename from data_processing/sandbox/widebeam_antennas_iq_to_rawacf.py rename to postprocessors/sandbox/widebeam_antennas_iq_to_bfiq.py index e357527..0f73fd2 100644 --- a/data_processing/sandbox/widebeam_antennas_iq_to_rawacf.py +++ b/postprocessors/sandbox/widebeam_antennas_iq_to_bfiq.py @@ -8,11 +8,10 @@ from typing import Union import numpy as np -from data_processing.convert_base import BaseConvert -from data_processing.antennas_iq_to_bfiq import ProcessAntennasIQ2Bfiq +from postprocessors import BaseConvert, AntennasIQ2Bfiq -class ProcessWidebeamAntennasIQ2Bfiq(BaseConvert): +class WidebeamAntennasIQ2Bfiq(BaseConvert): """ Class for conversion of Borealis antennas_iq files into rawacf files for beam-broadening experiments. This class inherits from BaseConvert, which handles all functionality generic to postprocessing borealis files. @@ -69,8 +68,8 @@ def __init__(self, infile: str, outfile: str, infile_structure: str, outfile_str if beam_azms is None: # Add extra phases here # STD_16_BEAM_ANGLE from superdarn_common_fields - self.beam_azms = [-26.25, -22.75, -19.25, -15.75, -12.25, -8.75, -5.25, -1.75, - 1.75, 5.25, 8.75, 12.25, 15.75, 19.25, 22.75, 26.25] + self.beam_azms = [-24.3, -21.06, -17.82, -14.58, -11.34, -8.1, -4.86, -1.62, 1.62, 4.86, 8.1, 11.34, 14.58, + 17.82, 21.06, 24.3] # STD_16_FORWARD_BEAM_ORDER self.beam_nums = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] @@ -105,18 +104,18 @@ def process_record(record: OrderedDict, averaging_method: Union[None, str], **kw record: OrderedDict hdf5 record, with new fields required by rawacf data format """ - record['first_range'] = ProcessAntennasIQ2Bfiq.calculate_first_range(record) - record['first_range_rtt'] = ProcessAntennasIQ2Bfiq.calculate_first_range_rtt(record) - record['lags'] = ProcessAntennasIQ2Bfiq.create_lag_table(record) - record['range_sep'] = ProcessAntennasIQ2Bfiq.calculate_range_separation(record) - record['num_ranges'] = ProcessAntennasIQ2Bfiq.get_number_of_ranges(record) + record['first_range'] = AntennasIQ2Bfiq.calculate_first_range(record) + record['first_range_rtt'] = AntennasIQ2Bfiq.calculate_first_range_rtt(record) + record['lags'] = AntennasIQ2Bfiq.create_lag_table(record) + record['range_sep'] = AntennasIQ2Bfiq.calculate_range_separation(record) + record['num_ranges'] = AntennasIQ2Bfiq.get_number_of_ranges(record) record['beam_azms'] = np.float64(kwargs['beam_azms']) record['beam_nums'] = np.uint32(kwargs['beam_nums']) - record['data'] = ProcessAntennasIQ2Bfiq.beamform_data(record) - record['data_descriptors'] = ProcessAntennasIQ2Bfiq.get_data_descriptors() - record['data_dimensions'] = ProcessAntennasIQ2Bfiq.get_data_dimensions(record) - record['antenna_arrays_order'] = ProcessAntennasIQ2Bfiq.change_antenna_arrays_order() + record['data'] = AntennasIQ2Bfiq.beamform_data(record) + record['data_descriptors'] = AntennasIQ2Bfiq.get_data_descriptors() + record['data_dimensions'] = AntennasIQ2Bfiq.get_data_dimensions(record) + record['antenna_arrays_order'] = AntennasIQ2Bfiq.change_antenna_arrays_order() return record diff --git a/exceptions/__init__.py b/postprocessors/utils/__init__.py similarity index 100% rename from exceptions/__init__.py rename to postprocessors/utils/__init__.py diff --git a/postprocessors/utils/filename_conversions.py b/postprocessors/utils/filename_conversions.py new file mode 100644 index 0000000..2bf4a06 --- /dev/null +++ b/postprocessors/utils/filename_conversions.py @@ -0,0 +1,73 @@ +""" +Copyright SuperDARN Canada 2022 + +Functions for converting a Borealis filename into a new filename for a downstream or +restructured file. +""" + + +def borealis_to_borealis_rename(filename, new_type, new_structure): + """ + Returns a typical Borealis-formatted filename with the new type and structure. + Typical Borealis-formatted filenames are YYYYMMDD.HHMM.SS.radar_id.slice_num.type.hdf5[.site] + where + radar id = three letter code, e.g. 'sas' + slice_num = number, e.g. 0 + type = borealis data type, one of 'antennas_iq', 'bfiq', or 'rawacf' + .site = optional file ending + + Parameters + ---------- + filename: str + Name of original file + new_type: str + New Borealis data type. 'antennas_iq', 'bfiq', or 'rawacf'. + new_structure: str + New Borealis file structure. 'site' or 'array' + + Returns + ------- + new_name: str + Newly formatted Borealis filename + """ + fields = filename.split(".") + if new_structure == 'site' and fields[-1] != 'site': + fields.append('site') + elif new_structure == 'array' and fields[-1] != 'hdf5': + fields.pop(-1) + + new_name = '.'.join(fields[0:5] + [new_type] + fields[6:]) + + return new_name + + +def borealis_to_sdarn_rename(filename, new_type): + """ + Returns a typical SuperDARN-formatted filename with the new type and structure. + Typical Borealis-formatted filenames are YYYYMMDD.HHMM.SS.radar_id.slice_num.type.hdf5[.site] + Typical SuperDARN-formatted filenames are YYYYMMDD.HHMM.SS.radar_id.alphabetic_slice_id.type + where + radar id = three letter code, e.g. 'sas' + slice_num = number, e.g. 0 + alphabetic_slice_id = single letter id, so if slice_num = 0, this = 'a', 1->'b', 2->'c', etc. + type = data type, one of 'antennas_iq', 'bfiq', or 'rawacf' for Borealis files, + one of 'iqdat', 'rawacf' for SuperDARN-formatted files. + .site = optional file ending + + Parameters + ---------- + filename: str + Name of original file + new_type: str + New Borealis data type. 'iqdat' or 'rawacf' + + Returns + ------- + new_name: str + Newly formatted Borealis filename + """ + fields = filename.split('.') + slice_id = fields[4] + slice_char = chr(int(slice_id) + ord('a')) + sdarn_file = '.'.join(fields[0:4] + [slice_char, new_type]) + return sdarn_file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..eb78848 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ['setuptools'] +build-backend = 'setuptools.build_meta' + +[project] +name = 'borealis-postprocessors' +version = '1.0.0' +description = 'Post-process SuperDARN Borealis data files' +readme = 'README.md' +authors = [{name = 'Remington Rohel'}] +license = {file = 'LICENSE'} +classifiers = [ + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python :: 3.6 :: 3.7 :: 3.8 :: 3.9', + 'Topic :: Scientific/Engineering', + 'Operating System :: Linux', +] +keywords = ['SuperDARN', 'Borealis'] +dependencies = ['numpy', 'pyDARNio', 'h5py', 'scipy', ] +requires-python = '>=3.6' + +[project.optional-dependencies] +gpu = ['cupy'] + +[project.urls] +Homepage = 'https://github.com/SuperDARNCanada/borealis_postprocessors' diff --git a/scripts/bistatic_postprocessing.py b/scripts/bistatic_postprocessing.py new file mode 100644 index 0000000..ca1340f --- /dev/null +++ b/scripts/bistatic_postprocessing.py @@ -0,0 +1,125 @@ +import argparse +import glob +import os +import h5py +import deepdish as dd +from datetime import datetime + +from postprocessors import ConvertFile, borealis_to_borealis_rename +from postprocessors.sandbox.widebeam_antennas_iq_to_bfiq import WidebeamAntennasIQ2Bfiq +from postprocessors.sandbox.bistatic_processing import BistaticProcessing + + +def main(in_directory: str, out_directory: str, out_struct: str, search_pattern: str, timestamp_dir: str): + """ + Postprocess all widebeam experiments from in_directory. + + Parameters + ---------- + in_directory: str + Path to directory containing widebeam experiment files. + out_directory: str + Path to directory to save post-processed widebeam experiment files. + out_struct: str + Final structure to save files as. Either 'site', 'array', or 'dmap' + search_pattern: str + Pattern to match when finding files + timestamp_dir: str + Path to directory to search for timestamp files. + + Returns + ------- + + """ + averaging_method = "mean" + + for path in glob.glob(f'{in_directory}/{search_pattern}'): + if not os.path.isfile(path): + continue + if path.endswith(".site"): + input_structure = "site" + else: + input_structure = "array" + + filename = os.path.basename(path) + + bfiq_file = borealis_to_borealis_rename(filename, 'bfiq', out_struct) + rawacf_site = borealis_to_borealis_rename(filename, 'rawacf', 'site') + rawacf_array = borealis_to_borealis_rename(filename, 'rawacf', 'array') + + bfiq_path = f'{out_directory}/{bfiq_file}' + rawacf_site_path = f'{out_directory}/{rawacf_site}' + rawacf_array_path = f'{out_directory}/{rawacf_array}' + + if out_struct == 'site': + rawacf_out = rawacf_site_path + else: + rawacf_out = rawacf_array_path + + start = datetime.utcnow() + + if os.path.isfile(rawacf_site_path) or os.path.isfile(rawacf_array_path): + # Either one is fine, we aren't picky + print(f'{path} - Already done. ', end='') + + elif os.path.isfile(bfiq_path): + # bfiq already exists + print(f'{bfiq_path} -> {rawacf_out} ', end='') + ConvertFile(bfiq_path, rawacf_out, 'bfiq', 'rawacf', 'site', out_struct, averaging_method) + os.remove(bfiq_path) # Don't want to keep this around + + else: + # Figure out how to process the file. Some experiments are straightforward, some need some love + if input_structure == 'site': + with h5py.File(path, 'r') as f: + records = sorted(list(f.keys())) + first_record = dd.io.load(path, f'/{records[0]}') + experiment_name = first_record['experiment_name'] + experiment_comment = first_record['experiment_comment'] + else: + with h5py.File(path, 'r') as f: + attributes = f.attrs + experiment_name = attributes['experiment_name'].decode('utf-8') + experiment_comment = attributes['experiment_comment'].decode('utf-8') + + if experiment_name in ['Widebeam_2tx', 'Widebeam_3tx', 'MultifreqWidebeam']: # These ones need some love + print(f'{path} -> {rawacf_out} ', end='') + WidebeamAntennasIQ2Bfiq(path, bfiq_path, input_structure, 'site') + ConvertFile(bfiq_path, rawacf_out, 'bfiq', 'rawacf', 'site', out_struct, averaging_method) + os.remove(bfiq_path) # We don't need to keep these around + + elif experiment_name == 'BistaticTest': + if 'Bistatic widebeam mode' in experiment_comment: + print(f'{path} - Bistatic listening experiment. ', end='') + # Search for file with timestamps to match against, then process. + fields = filename.split('.') + timestamp_pattern = f'{fields[0]}.{fields[1][0:2]}*timestamps*' # YYYYMMDD.HH*timestamps* + timestamp_files = glob.glob(f'{timestamp_dir}/{timestamp_pattern}') + if len(timestamp_files) == 0: + print('Error - no timestamp file. ', end='') + continue + else: # Expect 1 file + print(f'Using timestamps from {timestamp_files[0]}. ', end='') + BistaticProcessing(path, rawacf_out, input_structure, out_struct, timestamp_files[0]) + else: + print(f'{path} -> {rawacf_out} ', end='') + ConvertFile(path, rawacf_out, 'antennas_iq', 'rawacf', input_structure, out_struct, averaging_method) + else: + print(f'{path} - Not a widebeam experiment. ', end='') + + end = datetime.utcnow() + duration = (end - start).total_seconds() + print(f'Time: {duration:.2f} s') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('in_directory', type=str, help='Directory to search for files') + parser.add_argument('out_directory', type=str, help='Path to save output files') + parser.add_argument('--out_structure', type=str, help='Structure to save rawacf files as', default='site') + parser.add_argument('--pattern', type=str, help='Pattern to match when searching for files', + default='*antennas_iq*') + parser.add_argument('--timestamp_dir', type=str, help='Path to directory with timestamp files') + args = parser.parse_args() + + main(args.in_directory, args.out_directory, args.out_structure, args.pattern, args.timestamp_dir) diff --git a/scripts/memory_limited_postprocessing.py b/scripts/memory_limited_postprocessing.py new file mode 100644 index 0000000..8cbd920 --- /dev/null +++ b/scripts/memory_limited_postprocessing.py @@ -0,0 +1,85 @@ +import argparse +import glob +import os +from datetime import datetime + +from postprocessors import borealis_to_borealis_rename, Bfiq2Rawacf, AntennasIQ2Bfiq + + +def main(in_directory: str, out_directory: str, out_struct: str, search_pattern: str): + """ + Postprocess all widebeam experiments from in_directory. + + Parameters + ---------- + in_directory: str + Path to directory containing widebeam experiment files. + out_directory: str + Path to directory to save post-processed widebeam experiment files. + out_struct: str + Final structure to save files as. Either 'site', 'array', or 'dmap' + search_pattern: str + Pattern to match when finding files + + Returns + ------- + + """ + averaging_method = "mean" + + for path in glob.glob(f'{in_directory}/{search_pattern}'): + if not os.path.isfile(path): + continue + if path.endswith(".site"): + input_structure = "site" + else: + input_structure = "array" + + filename = os.path.basename(path) + + bfiq_file = borealis_to_borealis_rename(filename, 'bfiq', 'site') + rawacf_site = borealis_to_borealis_rename(filename, 'rawacf', 'site') + rawacf_array = borealis_to_borealis_rename(filename, 'rawacf', 'array') + + bfiq_path = f'{out_directory}/{bfiq_file}' + rawacf_site_path = f'{out_directory}/{rawacf_site}' + rawacf_array_path = f'{out_directory}/{rawacf_array}' + + if out_struct == 'site': + rawacf_out = rawacf_site_path + else: + rawacf_out = rawacf_array_path + + start = datetime.utcnow() + + if os.path.isfile(rawacf_site_path) or os.path.isfile(rawacf_array_path): + # Either one is fine, we aren't picky + print(f'{path} - Already done. ', end='') + + elif os.path.isfile(bfiq_path): + # bfiq already exists + print(f'{bfiq_path} -> {rawacf_out} ', end='') + Bfiq2Rawacf(bfiq_path, rawacf_out, 'site', out_struct, averaging_method, num_processes=1) + os.remove(bfiq_path) # Don't want to keep this around + + else: + print(f'{path} -> {bfiq_path} ', end='') + AntennasIQ2Bfiq(path, bfiq_path, input_structure, 'site', num_processes=1) + print(f'-> {os.path.basename(rawacf_out)} ', end='') + Bfiq2Rawacf(bfiq_path, rawacf_out, 'site', out_struct, averaging_method, num_processes=1) + + end = datetime.utcnow() + duration = (end - start).total_seconds() + print(f'Time: {duration:.2f} s') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('in_directory', type=str, help='Directory to search for files') + parser.add_argument('out_directory', type=str, help='Path to save output files') + parser.add_argument('--out_structure', type=str, help='Structure to save rawacf files as', default='site') + parser.add_argument('--pattern', type=str, help='Pattern to match when searching for files', + default='*antennas_iq*') + args = parser.parse_args() + + main(args.in_directory, args.out_directory, args.out_structure, args.pattern) diff --git a/scripts/timestamp_extraction.py b/scripts/timestamp_extraction.py new file mode 100644 index 0000000..343106f --- /dev/null +++ b/scripts/timestamp_extraction.py @@ -0,0 +1,73 @@ +import argparse +import glob +import os +from datetime import datetime + +import deepdish as dd + +from postprocessors.sandbox.extract_timestamps import ExtractTimestamps + + +def create_timestamp_file(infile: str, outfile: str): + new_group = {} + timestamps = dd.io.load(infile, '/sqn_timestamps') + num_sqns = dd.io.load(infile, '/num_sequences') + for i in range(num_sqns.shape[0]): + # key is the timestamp in seconds past epoch, value is the array + new_group[f'{timestamps[i, 0] * 1000:.0f}'] = {'data': {'sqn_timestamps': timestamps[i, :num_sqns[i]]}} + + dd.io.save(outfile, new_group) + + +def main(in_directory: str, out_directory: str, search_pattern: str): + """ + Extract timestamps from files matching search_pattern in in_directory into timestamp files in out_directory. + + Parameters + ---------- + in_directory: str + Path to directory containing widebeam experiment files. + out_directory: str + Path to directory to save post-processed widebeam experiment files. + search_pattern: str + Pattern to match when finding files + """ + + for path in glob.glob(f'{in_directory}/{search_pattern}'): + if not os.path.isfile(path): + continue + if path.endswith(".site"): + input_structure = "site" + else: + input_structure = "array" + + filename = os.path.basename(path) + fields = filename.split('.') + timestamp_file = '.'.join(fields[:5] + ['timestamps']) + timestamp_path = f'{out_directory}/{timestamp_file}' + + start = datetime.utcnow() + + if os.path.isfile(timestamp_path): + print(f'{path} - Already done. ', end='') + + else: + # The input file type (antennas_iq, bfiq, rawacf) doesn't matter, so just using antennas_iq as placeholder + # We have to process it downstream so the process_record method is called, so output_type is 'bfiq' + # ExtractTimestamps(path, timestamp_path, 'antennas_iq', 'bfiq', input_structure, 'site') + create_timestamp_file(path, timestamp_path) + + end = datetime.utcnow() + duration = (end - start).total_seconds() + print(f'Time: {duration:.2f} s') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('in_directory', type=str, help='Directory to search for files') + parser.add_argument('out_directory', type=str, help='Path to save output files') + parser.add_argument('--pattern', type=str, help='Pattern to match when searching for files', + default='*antennas_iq*') + args = parser.parse_args() + + main(args.in_directory, args.out_directory, args.pattern) diff --git a/scripts/widebeam_processing.py b/scripts/widebeam_processing.py new file mode 100644 index 0000000..6c31932 --- /dev/null +++ b/scripts/widebeam_processing.py @@ -0,0 +1,51 @@ +import glob +import os +import argparse + +import postprocessors +from postprocessors import ConvertFile +from postprocessors.sandbox.hamming_corrected import HammingBeamformingCorrected + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('indir', help='Directory containing files to process') + parser.add_argument('outdir', help='Directory to store processed files in') + parser.add_argument('--pattern', help='Pattern to search for when globbing files from indir', + default='*antennas_iq.hdf5.site') + parser.add_argument('--dmap', help='Process files to DMAP?', action='store_true') + args = parser.parse_args() + in_directory = args.indir + out_directory = args.outdir + pattern = args.pattern + + output_structure = 'site' + + for path in glob.glob(f'{in_directory}/{pattern}'): + if "antennas_iq" not in path: + continue + if not os.path.isfile(path): + continue + if path.endswith(".site"): + input_structure = "site" + else: + input_structure = "array" + + filename = os.path.basename(path) + + rawacf_file = postprocessors.borealis_to_borealis_rename(filename, 'rawacf', 'site') + dmap_file = postprocessors.borealis_to_sdarn_rename(filename, 'rawacf') + rawacf_path = out_directory + '/' + rawacf_file + dmap_path = out_directory + '/' + dmap_file + print(f'{path}') + + # Process the file to rawacf + if not os.path.isfile(rawacf_path): + print(f'\t-> {rawacf_path}') + processor = HammingBeamformingCorrected(path, rawacf_path, input_structure, output_structure, num_processes=5) + processor.process_file() + + # Process the file to dmap + if args.dmap and not os.path.isfile(dmap_path): + print(f'\t-> {dmap_path}') + ConvertFile(rawacf_path, dmap_path, "rawacf", "rawacf", 'site', 'dmap') + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f930bdb --- /dev/null +++ b/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup + +setup( + name='borealis-postprocessors', + version='1.0.0', + description='Post-process SuperDARN Borealis data files', + author='Remington Rohel', + license='GPLv3', + requires=['numpy', 'scipy', 'pyDARNio', 'h5py', 'deepdish'] +) diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..9b47633 --- /dev/null +++ b/test/README.md @@ -0,0 +1,8 @@ +# Test + +This folder contains both sample data files and testing scripts. + +## Data Files + +Files are named according to their data type and storage format. Additionally, there are two alternate rawacf files +which have far-range lag0 data replaced with alternate lag0 data. \ No newline at end of file diff --git a/test/test_antiq2bfiq.py b/test/test_antiq2bfiq.py index ddd0aaa..fd254e9 100644 --- a/test/test_antiq2bfiq.py +++ b/test/test_antiq2bfiq.py @@ -2,35 +2,36 @@ # Testing antennas_iq to bfiq conversion import os -import conversion +import postprocessors as pp from test.utils.compare_files import compare_files -array_infile = 'antennas_iq.hdf5' -site_infile = 'antennas_iq.hdf5.site' -site_outfile = 'test_antiq2bfiq.hdf5.site' -array_outfile = 'test_antiq2bfiq.hdf5' +if __name__ == '__main__': + array_infile = 'antennas_iq.hdf5' + site_infile = 'antennas_iq.hdf5.site' + site_outfile = 'test_antiq2bfiq.hdf5.site' + array_outfile = 'test_antiq2bfiq.hdf5' -compare_site_file = 'bfiq.hdf5.site' -compare_array_file = 'bfiq.hdf5' + compare_site_file = 'bfiq.hdf5.site' + compare_array_file = 'bfiq.hdf5' -# Convert from site file to both site and array files -conversion.ConvertFile(site_infile, site_outfile, 'antennas_iq', 'bfiq', 'site', 'site') -compare_files(compare_site_file, site_outfile) + # Convert from site file to both site and array files + pp.ConvertFile(site_infile, site_outfile, 'antennas_iq', 'bfiq', 'site', 'site') + compare_files(compare_site_file, site_outfile) -conversion.ConvertFile(site_infile, array_outfile, 'antennas_iq', 'bfiq', 'site', 'array') -compare_files(compare_array_file, array_outfile) + pp.ConvertFile(site_infile, array_outfile, 'antennas_iq', 'bfiq', 'site', 'array') + compare_files(compare_array_file, array_outfile) -# Remove the generated files -os.remove(site_outfile) -os.remove(array_outfile) + # Remove the generated files + os.remove(site_outfile) + os.remove(array_outfile) -# Convert from array fil to both site and array files -conversion.ConvertFile(array_infile, site_outfile, 'antennas_iq', 'bfiq', 'array', 'site') -compare_files(compare_site_file, site_outfile) + # Convert from array file to both site and array files + pp.ConvertFile(array_infile, site_outfile, 'antennas_iq', 'bfiq', 'array', 'site') + compare_files(compare_site_file, site_outfile) -conversion.ConvertFile(array_infile, array_outfile, 'antennas_iq', 'bfiq', 'array', 'array') -compare_files(compare_array_file, array_outfile) + pp.ConvertFile(array_infile, array_outfile, 'antennas_iq', 'bfiq', 'array', 'array') + compare_files(compare_array_file, array_outfile) -# Remove the generate files -os.remove(site_outfile) -os.remove(array_outfile) + # Remove the generate files + os.remove(site_outfile) + os.remove(array_outfile) diff --git a/test/test_antiq2rawacf.py b/test/test_antiq2rawacf.py index db902be..1e86818 100644 --- a/test/test_antiq2rawacf.py +++ b/test/test_antiq2rawacf.py @@ -2,35 +2,36 @@ # Testing antennas_iq to rawacf conversion import os -import conversion +import postprocessors as pp from test.utils.compare_files import compare_files -array_infile = 'antennas_iq.hdf5' -site_infile = 'antennas_iq.hdf5.site' -site_outfile = 'test_antiq2rawacf.hdf5.site' -array_outfile = 'test_antiq2rawacf.hdf5' +if __name__ == '__main__': + array_infile = 'antennas_iq.hdf5' + site_infile = 'antennas_iq.hdf5.site' + site_outfile = 'test_antiq2rawacf.hdf5.site' + array_outfile = 'test_antiq2rawacf.hdf5' -compare_site_file = 'rawacf.alt.hdf5.site' -compare_array_file = 'rawacf.alt.hdf5' + compare_site_file = 'rawacf.alt.hdf5.site' + compare_array_file = 'rawacf.alt.hdf5' -# Convert from site file to both site and array files -conversion.ConvertFile(site_infile, site_outfile, 'antennas_iq', 'rawacf', 'site', 'site') -compare_files(compare_site_file, site_outfile) + # Convert from site file to both site and array files + pp.ConvertFile(site_infile, site_outfile, 'antennas_iq', 'rawacf', 'site', 'site') + compare_files(compare_site_file, site_outfile) -conversion.ConvertFile(site_infile, array_outfile, 'antennas_iq', 'rawacf', 'site', 'array') -compare_files(compare_array_file, array_outfile) + pp.ConvertFile(site_infile, array_outfile, 'antennas_iq', 'rawacf', 'site', 'array') + compare_files(compare_array_file, array_outfile) -# Remove the generated files -os.remove(site_outfile) -os.remove(array_outfile) + # Remove the generated files + os.remove(site_outfile) + os.remove(array_outfile) -# Convert from array fil to both site and array files -conversion.ConvertFile(array_infile, site_outfile, 'antennas_iq', 'rawacf', 'array', 'site') -compare_files(compare_site_file, site_outfile) + # Convert from array file to both site and array files + pp.ConvertFile(array_infile, site_outfile, 'antennas_iq', 'rawacf', 'array', 'site') + compare_files(compare_site_file, site_outfile) -conversion.ConvertFile(array_infile, array_outfile, 'antennas_iq', 'rawacf', 'array', 'array') -compare_files(compare_array_file, array_outfile) + pp.ConvertFile(array_infile, array_outfile, 'antennas_iq', 'rawacf', 'array', 'array') + compare_files(compare_array_file, array_outfile) -# Remove the generate files -os.remove(site_outfile) -os.remove(array_outfile) + # Remove the generate files + os.remove(site_outfile) + os.remove(array_outfile) diff --git a/test/test_bfiq2rawacf.py b/test/test_bfiq2rawacf.py index befa5b5..e63b7f3 100644 --- a/test/test_bfiq2rawacf.py +++ b/test/test_bfiq2rawacf.py @@ -2,35 +2,36 @@ # Testing bfiq to rawacf conversion import os -import conversion +import postprocessors as pp from test.utils.compare_files import compare_files -array_infile = 'bfiq.hdf5' -site_infile = 'bfiq.hdf5.site' -site_outfile = 'test_bfiq2rawacf.hdf5.site' -array_outfile = 'test_bfiq2rawacf.hdf5' +if __name__ == '__main__': + array_infile = 'bfiq.hdf5' + site_infile = 'bfiq.hdf5.site' + site_outfile = 'test_bfiq2rawacf.hdf5.site' + array_outfile = 'test_bfiq2rawacf.hdf5' -compare_site_file = 'rawacf.alt.hdf5.site' -compare_array_file = 'rawacf.alt.hdf5' + compare_site_file = 'rawacf.alt.hdf5.site' + compare_array_file = 'rawacf.alt.hdf5' -# Convert from site file to both site and array files -conversion.ConvertFile(site_infile, site_outfile, 'bfiq', 'rawacf', 'site', 'site') -compare_files(compare_site_file, site_outfile) + # Convert from site file to both site and array files + pp.ConvertFile(site_infile, site_outfile, 'bfiq', 'rawacf', 'site', 'site') + compare_files(compare_site_file, site_outfile) -conversion.ConvertFile(site_infile, array_outfile, 'bfiq', 'rawacf', 'site', 'array') -compare_files(compare_array_file, array_outfile) + pp.ConvertFile(site_infile, array_outfile, 'bfiq', 'rawacf', 'site', 'array') + compare_files(compare_array_file, array_outfile) -# Remove the generated files -os.remove(site_outfile) -os.remove(array_outfile) + # Remove the generated files + os.remove(site_outfile) + os.remove(array_outfile) -# Convert from array fil to both site and array files -conversion.ConvertFile(array_infile, site_outfile, 'bfiq', 'rawacf', 'array', 'site') -compare_files(compare_site_file, site_outfile) + # Convert from array file to both site and array files + pp.ConvertFile(array_infile, site_outfile, 'bfiq', 'rawacf', 'array', 'site') + compare_files(compare_site_file, site_outfile) -conversion.ConvertFile(array_infile, array_outfile, 'bfiq', 'rawacf', 'array', 'array') -compare_files(compare_array_file, array_outfile) + pp.ConvertFile(array_infile, array_outfile, 'bfiq', 'rawacf', 'array', 'array') + compare_files(compare_array_file, array_outfile) -# Remove the generate files -os.remove(site_outfile) -os.remove(array_outfile) + # Remove the generate files + os.remove(site_outfile) + os.remove(array_outfile) diff --git a/test/utils/compare_files.py b/test/utils/compare_files.py index 22057f2..df347d9 100644 --- a/test/utils/compare_files.py +++ b/test/utils/compare_files.py @@ -3,13 +3,13 @@ """ This file contains functions for comparing the contents of two HDF5 files. """ -import deepdish as dd +import h5py import numpy as np def compare_files(file1, file2): - group1 = dd.io.load(file1) - group2 = dd.io.load(file2) + group1 = h5py.File(file1, 'r') + group2 = h5py.File(file2, 'r') def compare_dictionaries(dict1, dict2, prefix): compare_string = "" @@ -19,6 +19,12 @@ def compare_dictionaries(dict1, dict2, prefix): uniq2 = keys2.difference(keys1) shared = keys1.intersection(keys2) + attrs1 = set(dict1.attrs.keys()) + attrs2 = set(dict2.attrs.keys()) + uniq_attrs1 = attrs1.difference(attrs2) + uniq_attrs2 = attrs2.difference(attrs1) + shared_attrs = attrs1.intersection(attrs2) + try: assert len(uniq1) == len(uniq2) == 0 except AssertionError: @@ -33,22 +39,26 @@ def compare_dictionaries(dict1, dict2, prefix): if type(entry1) != type(entry2): compare_string += prefix + f'Mismatched types for key {key}' # If they are dictionaries, recurse - elif type(entry1) == dict: + elif type(entry1) == h5py.Group: compare_dictionaries(entry1, entry2, prefix + '\t') # Otherwise, they must be lists or values, and so can be compared directly else: - equal = True - # Compare floating-point values differently if 'float' in str(entry1.dtype) or 'complex' in str(entry1.dtype): + if entry1.shape != entry2.shape: + entry1 = entry1[()].reshape(entry2.shape) + if not np.allclose(entry1, entry2, equal_nan=True): - equal = False + compare_string += prefix + f"/{key}:\n" \ + f"\t{entry1}\n" \ + f"\t{entry2}\n" \ + f"\tDifference: " \ + f"{np.nanmax(np.abs((entry1[:] - entry2[:])/entry1[:]))}\n" # Comparing non-floating-point values elif not np.array_equal(entry1, entry2): - equal = False - - if not equal: - compare_string += prefix + f"/{key}\n" + compare_string += prefix + f"/{key}:\n" \ + f"\t{entry1}\n" \ + f"\t{entry2}\n" return compare_string