Wednesday, August 28, 2013

Scan Software Installed Powershell Script

The process has been design to scan on the windows based computer  installed software and logs it to text file and SQL Server database.   It requires the tblHardware_Populated and tblSoftware_Populated tables.  The following DML commands will create them.


CREATE TABLE [dbo].[tblHardware_Populated](
 [ComputerName] [nvarchar](30) NOT NULL,
 [SerialNumber] [nvarchar](50) NULL,
 [PhysicalMemory] [nvarchar](10) NOT NULL,
 [DiskSpace] [nvarchar](max) NOT NULL,
 [Domain] [nvarchar](50) NOT NULL,
 [Manufacturer] [nvarchar](100) NULL,
 [Model] [nvarchar](100) NULL,
 [PrimaryOwnerName] [nvarchar](50) NOT NULL,
 [DateCreated] [datetime] NOT NULL,
 [JobName] [nvarchar](256) NULL,
 [ActionsPath] [nvarchar](256) NULL,
 [LastRunTime] [datetime] NULL,
 [NextRunTime] [datetime] NULL,
 [Actions] [nvarchar](256) NULL,
 CONSTRAINT [PK_tblHardware_Populated] PRIMARY KEY CLUSTERED
(
 [ComputerName] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
GO
CREATE TABLE [dbo].[tblSoftware_Populated](
 [SoftwareID] [int] IDENTITY(1,1) NOT NULL,
 [ComputerName] [nvarchar](30) NOT NULL,
 [PackageName] [nvarchar](200) NOT NULL,
 [InstallDate] [date] NULL,
 [InstallLocation] [nvarchar](256) NULL,
 [PackageCache] [nvarchar](50) NULL,
 [Vendor] [nvarchar](200) NULL,
 [Version] [nvarchar](30) NULL,
 CONSTRAINT [PK_tblSoftware_Populated] PRIMARY KEY CLUSTERED
(
 [SoftwareID] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
GO
 ALTER TABLE [dbo].[tblSoftware_Populated]  WITH NOCHECK ADD  CONSTRAINT [FK_tblSoftware_Populated_tblHardware_Populated] FOREIGN KEY([ComputerName])

REFERENCES [dbo].[tblHardware_Populated] ([ComputerName])

GO
ALTER TABLE [dbo].[tblSoftware_Populated] NOCHECK CONSTRAINT [FK_tblSoftware_Populated_tblHardware_Populated]

GO

Application Folder Structure

The Universal Data Link file dbConnection.udl provides connection to SQL Server database and Powershell script should be located in .\InstalledSoftware\Script\ folder.  Output  data and scan.log (process log) file will be created in .\InstalledSoftware\Output\ directory.  Here is the script listing:
#This script scans installed software and log it to text file and ConfigManag database
    function datetime{
      $year = (Get-Date).Year;
      $month = Get-Date -format MM;
      $day = Get-Date -format dd;
      $hour = Get-Date -format hh;
      $minute = Get-Date -format mm;
      $second = Get-Date -format ss;
      return "$day/$month/$year $hour`:$minute`:$second";
    }
  
    function getTasks($path) {
        $out = @()
        # Get root tasks
        $schedule.GetFolder($path).GetTasks(0) | % {
            $xml = [xml]$_.xml
            $out += New-Object psobject -Property @{
                "Name" = $_.Name
                "Path" = $_.Path
                "LastRunTime" = $_.LastRunTime
                "NextRunTime" = $_.NextRunTime
                "Actions" = ($xml.Task.Actions.Exec | % { "$($_.Command) $($_.Arguments)" }) -join "`n"
            }
        }
        # Get tasks from subfolders
        $schedule.GetFolder($path).GetFolders(0) | % {
            $out += getTasks($_.Path)
        }
        #Output
        $out
    }
    $tasks = @()
    $schedule = New-Object -ComObject "Schedule.Service"
    $schedule.Connect()
    # Start inventory
    $tasks += getTasks("\")
    # Close com
    [System.Runtime.Interopservices.Marshal]::ReleaseComObject($schedule) | Out-Null
    Remove-Variable schedule
    # Output all tasks
    #$tasks
    CLEAR-HOST
    [void][reflection.assembly]::LoadWithPartialName("microsoft.visualbasic")
    [console]::ForegroundColor = "yellow"
    $date = datetime;
    "** Start process $date **";
    write-host "`r";
    $dt = Get-Date -format "yyyy_MM_dd"
    $a = $ErrorActionPreference
    $ErrorActionPreference = "SilentlyContinue"
    try{
        $file =  "Scan_" + $dt + ".txt"
        $JobName = "Scan Software Installed"
        $folder = ".\InstalledSoftware\Output\"
        $filePath = $folder + $file
        $outlog = $folder+ "\scan.log"
        $($date) | out-file $outlog -append
        $testFileExists = Test-Path $outlog
        if ($testFileExists -eq $false) {
            "" | out-file $outlog
        }
        $scriptPath = split-path -parent $MyInvocation.MyCommand.Definition
        $myconnectionstring = "File Name=" + $scriptPath + "\dbConnection.udl"
      
        $adOpenStatic = 3
  $adLockOptimistic = 3
        $objConnection = New-Object -comobject ADODB.Connection
        $objRecordset = New-Object -comobject ADODB.Recordset
        $objConnection.Open($myconnectionstring)
        if($objConnection.state -eq 0){
          "Could not establish connection" | out-file $outlog -append
          exit
        }
      
        $cmd = Get-ChildItem $folder -force | select name
        $testFileExists = $false
        $filesNo = 0
        foreach ($str in $cmd){
    $test = $str.name
                if ($str.name -eq $file) {
                    $testFileExists = $true
                    break
                } else {
                    $StartsWith = $test -match  "Scan"
                    if ($StartsWith -eq $true){
                        $filesNo = $filesNo + 1
                    }
                }  
        }

  #Delete existing data        if ($testFileExists -eq $false) {
            if ($filesNo -gt 0){
    Get-ChildItem $folder | Remove-Item -r -force
            }  
            New-Item $filePath -type file
        
         $strSQL = "TRUNCATE TABLE dbo.tblSoftware_Populated"
         $retval = $objConnection.Execute($strSQL)
            $strSQL = "ALTER TABLE dbo.tblSoftware_Populated DROP CONSTRAINT FK_tblSoftware_Populated_tblHardware_Populated"
            $retval = $objConnection.Execute($strSQL)
            $strSQL = "TRUNCATE TABLE dbo.tblHardware_Populated"
         $retval = $objConnection.Execute($strSQL)
            $strSQL = "ALTER TABLE dbo.tblSoftware_Populated  WITH NOCHECK ADD  CONSTRAINT FK_tblSoftware_Populated_tblHardware_Populated FOREIGN KEY(ComputerName)"
            $strSQL = $strSQL + "REFERENCES dbo.tblHardware_Populated (ComputerName)"
            $retval = $objConnection.Execute($strSQL)
            $strSQL = "ALTER TABLE dbo.tblSoftware_Populated NOCHECK CONSTRAINT FK_tblSoftware_Populated_tblHardware_Populated"
            $retval = $objConnection.Execute($strSQL)
        } 
        Add-Content $filePath $log
        $log = "*" * 50
        Add-Content $filePath $log  #Check if it is already scanned        $strSQL = "SELECT COUNT(*) counter FROM dbo.tblHardware_Populated WHERE"
        $strSQL = $strSQL + " ComputerName = '" + $($env:computername) + "'"
        $objRecordset.Open($strSQL,$objConnection,$adOpenStatic,$adLockOptimistic)
        if($objRecordset.EOF -eq $True){
     "No Data found" | out-file $outlog -append
     exit
  }
        $counter = $objRecordset.Fields.Item("counter").Value
        $objRecordset.Close()  #Populate Hardware        if ($counter -eq 0){
            $strSQL = "INSERT INTO dbo.tblHardware_Populated("
            $strSQL = $strSQL + "ComputerName"
            $strSQL = $strSQL + ",SerialNumber"
            $strSQL = $strSQL + ",PhysicalMemory"
            $strSQL = $strSQL + ",DiskSpace"
            $strSQL = $strSQL + ",Domain"
            $strSQL = $strSQL + ",Manufacturer"
            $strSQL = $strSQL + ",Model"
            $strSQL = $strSQL + ",PrimaryOwnerName"
            $strSQL = $strSQL + ",DateCreated"
            $strSQL = $strSQL + ",JobName"
            $strSQL = $strSQL + ",ActionsPath"
            $strSQL = $strSQL + ",LastRunTime"
            $strSQL = $strSQL + ",NextRunTime"
            $strSQL = $strSQL + ",Actions"
            $strSQL = $strSQL + " )VALUES("
            $strSQL = $strSQL + "'" + $($env:computername) + "'"
            $log = "Computer Name: " +$env:computername
            Add-Content $filePath $log
            $log = get-wmiobject win32_bios | select-object serialnumber
            foreach ($str in $log){
                $out = "Serial Number: " + $str.serialnumber
                $strSQL = $strSQL + ",'" + $($str.serialnumber) + "'"
                Add-Content $filePath $out
            }
          
            $txt = Get-WMIObject -class Win32_PhysicalMemory |
            Measure-Object -Property capacity -Sum |
            select @{N="~"; E={[math]::round(($_.Sum / 1GB),2)}}
            $log = $txt -replace "@{",""
            $log = $log -replace "}",""
            $log = $log -replace "=",""
            $log = $log -replace "~",""
            $strSQL = $strSQL + ",'" + $log + "'"
            $log = "Total Physical Ram: " + $log
            Add-Content $filePath $log
            $txt = ""
            $log = gwmi win32_volume -Filter 'drivetype = 3' |
            select driveletter, label, @{LABEL='GBfreespace';EXPRESSION={"{0:N2}" -f ($_.freespace/1GB)} }
            foreach ($str in $log){
                $out = "Driveletter : " + $str.driveletter + "`t"
                $txt = $txt + $out;
                #Add-Content $filePath $out
                $out = "Label : " + $str.label + "`t"
                $txt = $txt + $out;
                #Add-Content $filePath $out
                $out = "GBfreespace : " + $str.GBfreespace + "`t"
                $txt = $txt + $out;
                #Add-Content $filePath $out
            }
            $strSQL = $strSQL + ",'" + $txt + "'"
            Add-Content $filePath $txt
            $log = get-wmiobject win32_computersystem
            foreach ($str in $log){
                $out = "Domain : " + $str.Domain
                $strSQL = $strSQL + ",'" + $($str.Domain) + "'"
                Add-Content $filePath $out
                $out = "Manufacturer : " + $str.Manufacturer
                $strSQL = $strSQL + ",'" + $($str.Manufacturer) + "'"
                Add-Content $filePath $out
                $out = "Model : " + $str.Model
                $strSQL = $strSQL + ",'" + $($str.Model) + "'"
                Add-Content $filePath $out
                $out = "PrimaryOwnerName : " + $str.PrimaryOwnerName
                $strSQL = $strSQL + ",'" + $($str.PrimaryOwnerName) + "'"
                Add-Content $filePath $out
            }
            $strSQL = $strSQL + ",CONVERT(datetime,'" + $date + "',103)"
          
            $out = ""
            foreach ($str in $tasks){
                if ($str.Name -eq $JobName){
                    $out = "Name: " + $str.Name
                    $strSQL = $strSQL + ",'" + $($str.Name) + "'"
                    Add-Content $filePath $out
                    $out = "Path: " + $str.Path
                    $strSQL = $strSQL + ",'" + $($str.Path) + "'"
                    Add-Content $filePath $out
                    $out = "LastRunTime: " + $str.LastRunTime
                    $strSQL = $strSQL + ", CONVERT(datetime,'" + $($str.LastRunTime) + "',101)"
                    Add-Content $filePath $out
                    $out = "NextRunTime: " + $str.NextRunTime
                    $strSQL = $strSQL + ", CONVERT(datetime,'" + $($str.NextRunTime) + "',101)"
                    Add-Content $filePath $out
                    $out = "Actions: " + $str.Actions
                    $strSQL = $strSQL + ",'" + $($str.Actions) + "'"
                    Add-Content $filePath $out
                    Break
                }
              
            }
          
            if ($out -eq ""){
                $out = "Name: "
                $strSQL = $strSQL + ",Null"
                Add-Content $filePath $out
                $out = "Path: "
                $strSQL = $strSQL + ",Null"
                Add-Content $filePath $out
                $out = "LastRunTime: "
                $strSQL = $strSQL + ",Null"
                Add-Content $filePath $out
                $out = "NextRunTime: "
                $strSQL = $strSQL + ",Null"
                Add-Content $filePath $out
                $out = "Actions: "
                $strSQL = $strSQL + ",Null"
                Add-Content $filePath $out
            }
          
            $strSQL = $strSQL + ")"
            $retval = $objConnection.Execute($strSQL)
            $log = "*" * 50
            Add-Content $filePath $log
            Add-Content $filePath ""
            $log = " " * 15 + "Software Installed"
            Add-Content $filePath $log
        
   #Populate Software            Add-Content $filePath ""
            $log = Get-WmiObject -Class Win32_Product -ComputerName . |
             select Name,InstallDate,InstallLocation,PackageCache,Vendor,Version,IdentifyingNumber
            foreach ($str in $log){
                $strSQL = "INSERT INTO dbo.tblSoftware_Populated("
                $strSQL = $strSQL + "ComputerName"
                $strSQL = $strSQL + ",PackageName"
                $strSQL = $strSQL + ",InstallDate"
                $strSQL = $strSQL + ",InstallLocation"
                $strSQL = $strSQL + ",PackageCache"
                $strSQL = $strSQL + ",Vendor"
                $strSQL = $strSQL + ",Version"
                $strSQL = $strSQL + " )VALUES("
                $strSQL = $strSQL + "'" + $($env:computername) + "'"
                $out = "Name: " + $str.Name
                $strSQL = $strSQL + ",'" + $($str.Name) + "'"
                Add-Content $filePath $out
                $dt = $str.InstallDate
                $dtd = ",null"
                if ($dt) {
                    $dta = $dt.Substring(6,2) + "/" + $dt.Substring(4,2) + "/" + $dt.Substring(0,4)
                    $dtd = ",'" + $dt + "'"
                }
                $out = "InstallDate: " + $dta
                $strSQL = $strSQL + $dtd
                Add-Content $filePath $out
                $loc = ",null"
                if ($str.InstallLocation) {
                    $loc = ",'" + $str.InstallLocation + "'"
                }  
                $out = "InstallLocation: " + $str.InstallLocation
                $strSQL = $strSQL + $loc
                Add-Content $filePath $out
                $out = "PackageCache: " + $str.PackageCache
                $strSQL = $strSQL + ",'" + $($str.PackageCache) + "'"
                Add-Content $filePath $out
                $out = "Vendor: " + $str.Vendor
                $strSQL = $strSQL + ",'" + $($str.Vendor) + "'"
                Add-Content $filePath $out
                $out = "Version: " + $str.Version
                $strSQL = $strSQL + ",'" + $($str.Version) + "'"
                Add-Content $filePath $out
                $strSQL = $strSQL + ")"
                Add-Content $filePath ""
                #Write-Output $strSQL
                $retval = $objConnection.Execute($strSQL)
            }
            $JobStatus = "OK"
        }  
    } catch [Exception] {
            "$($_.InvocationInfo.ScriptName)($($_.InvocationInfo.ScriptLineNumber)): $($_.InvocationInfo.Line)" | out-file $outlog -append
         $("Error catched: " + $_.Exception.GetType().FullName) | out-file $outlog -append
         $("Error catched: " + $_.Exception.Message) | out-file $outlog -append
         $JobStatus = "not OK"
         continue;
    }
    $ErrorActionPreference = $a
    $objConnection.Close()
    $date = datetime;
    "** End of process $date **";
 
The process can be scheduled according your requirements.

If you find this script useful please donate generously.

Monday, February 4, 2013

Exporting pdf documents properties to the text file

The following script extracts pdf documents some property fields and dump them to the text file.  Note that if itextsharp.dll does not exist on your system it can be downloaded from http://sourceforge.net/projects/itextsharp/.  To add other fields to output see documentation on http://www.adobe.com.

[System.Reflection.Assembly]::LoadFrom(\\your_server\..\itextsharp.dll)

function Get_PDF_Properties($path){
    Try{
        $raf = New-object iTextSharp.text.pdf.RandomAccessFileOrArray($path)
        $reader = New-object iTextSharp.text.pdf.PdfReader($raf, $Nothing)
        $myPDF= ""| select Title,Subject
        $mypdf.title=($reader).info.item("Title")
        $sub = 0
        if ((($reader).info.item("Subject").Length -ne 0)){
            $mypdf.subject=($reader).info.item("Subject")
            $sub = 1
        } 
    }
    Catch [Exception]{
        if ($sub -eq 0) {
            $mypdf.subject=""
        } else {
            Write-Host "Generic Exception"
            Write-Host $_
            $_ | Select *
        } 
    }
  return $mypdf
}
$in = \\your_server\..\
$out = "\\your_server\..\file_name.txt"
Add-Content -Path $out -Value "ImagePath`tTitle`tSubject" -Encoding UTF8
$dirstr = Get-ChildItem $in -recurse -filter "*.pdf" | Select-Object FullName
foreach ($objItem in $dirstr) {
    $path = $objItem -replace "@{FullName=", ""
    $path = $path -replace "}", ""
    $mypdf = Get_PDF_Properties($path)
    $pdf = $mypdf -replace "@{Title=", ""
    $pdf = $pdf -replace " Subject=", ""
    $pdf = $pdf -replace "}", ""
    $pdf = $pdf -replace ";", "`t"
    write-host "$path`t$pdf"
    Add-Content -Path $out -Value "$path`t$pdf" -Encoding UTF8
}
 
If you find this script useful please donate generously.

Thursday, December 13, 2012

Powershell script to Refresh Excel Data

The following sript opends Excel files one by one, refreshes the data, saves the file back and closes the file.


 Function RefreshExcelData($Excel){
  $App = new-object -comobject excel.application
  $App.Workbooks.Open($Excel)
  $App.Visible = $true
  $App.ActiveWorkbook.Refreshall()
  $App.Save()
  $App.Quit()
  $username = $env:username
  Remove-Item "C:\Users\$username\Documents\RESUME.XLW"
 }

 CLEAR-HOST
 $Host.UI.RawUI.ForegroundColor = "yellow"
 $Excel = 'R:\folder\file1.xlsx'
 RefreshExcelData $Excel
 $Excel = 'R:\folder\file2.xlsx'
 RefreshExcelData $Excel
 $Excel = 'R:\folder\file3.xlsx'
 RefreshExcelData $Excel
 $Host.UI.RawUI.ForegroundColor = "white"

If you find this script useful please donate generously.

Thursday, December 6, 2012

Powershell script to ftp MySQL backup file and restore it locally.

The following Powershell script can be customised and scheduled (

 powershell.exe -executionpolicy bypass -file E:\scripts\dbrestore.ps1
) on your server:

 <#########################################################################
  Purpose:  FTP down MySQL backup file and restore it locally.
  Requirements: 1) Source backup file automatically created daily.
       The name is generated as YYYYMMDD_dbname.sql
       and located in /dbbackup folder.
      2) Indy.Sockets.dll exits on the destination system.
       Lets presume that it is located in e:\scripts
 #########################################################################>
 function Open-FTPConnection($ftphost, $username, $password) {
  [void][Reflection.Assembly]::LoadFrom("e:\scripts\Indy.Sockets.dll");
   $ftp = new-object Indy.Sockets.FTP;
   $ftp.Disconnect();
   $ftp.Host = $ftphost;
   $ftp.Username = $username;
   $ftp.Password = $password;
   $ftp.Connect();
   $ftp.Passive=$true;
   return $ftp;
  }

 function Close-FTPConnection($ftp) {
  $ftp.Disconnect();
  }

 function Draw-DirectoryContents($ftp) {
  "Listing directory contents ..";
  $ls = new-object System.Collections.Specialized.StringCollection;
  $ftp.List($ls, "", $true);
  foreach($file in $ls){
   $file;
   }
   Get-FTPCurrentLocation $ftp
  "Complete";
  }

 function Get-FTPCurrentLocation($ftp) {
  return $ftp.RetrieveCurrentDir();
  }

 function Set-ParentLocation($ftp) {
  "Changing to Parent Directory ..";
  $ftp.ChangeDirUp();
  Draw-DirectoryContents;
  }

 function Download-FTPFile($ftp, $sourceFileName, $targetDir) {
  "Downloading {0} into {1}.." -f $sourceFileName, $targetDir;
  $ftp.Get($sourceFileName, ($targetDir + $sourceFileName), $true, $false);
  "Downloading of {0} into {1} is complete" -f $sourceFileName,
  [Environment]::CurrentDirectory;
  }
  
 function Upload-FTPFile($ftp, $sourceFileName, $targetDir) {
  "Uploading {0} into {1}.." -f $sourceFileName, $targetDir;
  $ftp.Put($sourceFileName, ($targetDir + $sourceFileName), $false);
  "Uploading of {0} into {1} is complete" -f $sourceFileName, $targetDir;
  }
 
 function Process-FTPItem($ftp, [string]$sel) {
   [string[]]$fields=[Regex]::Split($sel, " +");
   $startField=8; #the file/directory name starts after 8 fields
   [string]$name="";
  
  for ($field=$startField; $field -lt $fields.Length; $field++) {
    if ($field -eq $startField) {
    $temp = "";
    } else {
    $temp = " ";
    }
    $name += $temp + $fields[$field];
    #add aditional space for name split into multiple fields
   }
  
  if ($sel[0] -eq 'd') { #directory
    "Changing directoy to {0} .." -f $name;
    $ftp.ChangeDir($name);
    Draw-DirectoryContents $ftp #redraw contents after changing directory
  } else {
    if ($sel[0] -eq '-') { #plain file has '-' as first character
    return $name; #return the name of file to download
    }
  }
  }
 
 function datetime{
  $year = (Get-Date).Year;
  $month = Get-Date -format MM;
  $day = Get-Date -format dd;
  $hour = Get-Date -format hh;
  $minute = Get-Date -format mm;
  $second = Get-Date -format ss;
  return "$day/$month/$year $hour`:$minute`:$second";
 }

 $year = (Get-Date).Year;
 $month = Get-Date -format MM;
 $day = Get-Date -format dd;
 $hour = Get-Date -format hh;
 $minute = Get-Date -format mm;
 $second = Get-Date -format ss;
 CLEAR-HOST
 $Host.UI.RawUI.ForegroundColor = "yellow"
 [void][reflection.assembly]::LoadWithPartialName("microsoft.visualbasic")
 $date = datetime;
 "** Start process $date **";
 write-host "`r";
 $theDate = "$year$month$day";
 $word_to_delete = "_dbname.sql";
 $file = "/dbbackup/$theDate$word_to_delete";
 $dir = "e:\db_backup"; #Destination folder
 $from = "$dir\dbbackup";
 $to = "$dir\$theDate`_$hour$minute$second";
 $out = "$to\$theDate$word_to_delete";
 $procedures = "/dbbackup/curls_procedures_$year-$month-$day.sql"
 if (!(Test-Path -path "$dir\$theDate*")){
  if ((Test-Path -path "$dir\dbbackup")){
   Remove-Item "$dir\dbbackup";
  }
  New-Item "$dir\dbbackup" -type directory;
  $f = Open-FTPConnection "www.sitename.com" "ftpUserName" "ftpPassword";
  Download-FTPFile $f $file ($dir);
  Download-FTPFile $f $procedures ($dir);
  Close-FTPConnection $f;
  Rename-Item $from $to;
  #Global replace of the keyword
  (Get-Content $out) | Foreach-Object {$_ -replace "sourceDBname", "destinationDBname"} | Set-Content $out;
  Write-Host $to;
  Write-Host $from;
 } else {
  "File already downloaded!";
  "Processing local restore ...";
  $items = Get-ChildItem -Path "$dir\$theDate*"
  foreach ($item in $items){
   $val = $item.Attributes
     if ($item.Attributes -Like "Directory*"){
    $theDir = $item.Name
     }
  }
  $theDir = "$dir\$theDir"
  $fileEntries = [IO.Directory]::GetFiles($theDir); 
  foreach($filename in $fileEntries){ 
   $file = $filename;
  }
 }

 $mysql = "mysql -u dbUserName --password=dbPassword destinationDBname --execute=""SELECT CONCAT('DROP TABLE ',GROUP_CONCAT(TABLE_NAME),';') AS stmt FROM information_schema.TABLES WHERE  TABLE_SCHEMA = 'destinationDBname' INTO OUTFILE 'e:\\db_backup\\dropAllTables.sql'"""
 Write-Host $mysql;
 & cmd /c $mysql;
 $mysql = "mysql -u dbUserName --password=dbPassword destinationDBname < e:\db_backup\dropAllTables.sql"
 Write-Host $mysql;
 & cmd /c $mysql;
 $mysql = "mysql -u dbUserName --password=dbPassword destinationDBname < $out"
 Write-Host $mysql;
 write-host "`r";
 $date = datetime;
 "** End of process $date **";

If you find this script useful please donate generously.

Tuesday, June 5, 2012

Reading live XML file to the SQL Server database by combining consumed and unconsumed XML data.

Task: Query the XML (LicenseKeys.xml) file from MS SQL Server database.
XML file fragment:

<?xml version="1.0" encoding="utf-8"?>
<licenseBatch xmlns="http://site/schemas/licensekey">
                <license batchID="1" batchStatus="Active" expireDays="60" admin="WESTERN_SYDNEY\rsakhamuri" adminEmail="Ramya.Sakhamuri@site.com" requester="rsakhamuri" requesterEmail="Ramya.Sakhamuri@site.com" creationTimeStamp="2010-02-04T15:46:02.525392+11:00" activationType="OfflinePossible">
                                <note>test</note>
                                <licenseKey sequence="1" key="3fea9da9-28b1-47f1-82ff-4198c96330d7" owner="ramya" ownerEmail="" status="ReActivated" lastModified="2010-02-04T05:03:40.663936Z" noOfActivation="3" maxActivationCount="50">
                                                <activator IP="127.0.0.1" userName="WESTERN_SYDNEY\rsakhamuri" email="" activationTimeStamp="2010-02-04T15:48:57.9026368+11:00">rams</activator>
                                                <activator IP="127.0.0.1" userName="WESTERN_SYDNEY\rsakhamuri" email="" activationTimeStamp="2010-02-04T15:49:11.5434364+11:00">rams</activator>
                                                <activator IP="172.24.108.48" userName="WESTERN_SYDNEY\rsakhamuri" email="ramya.sakhamuri@site.com" activationTimeStamp="2010-02-04T16:03:40.663936+11:00">ramya</activator>
                                </licenseKey>
                                <licenseKey sequence="2" key="acbebeeb-af3e-40d1-a9ca-f03dd0a5cce4" owner="" ownerEmail="" status="InActive" lastModified="2010-02-04T15:46:02.525392+11:00" noOfActivation="0" maxActivationCount="50" />
                                <licenseKey sequence="3" key="8efcc7c4-c367-4316-b8b3-8f5c8cfc244c" owner="" ownerEmail="" status="InActive" lastModified="2010-02-04T15:46:02.525392+11:00" noOfActivation="0" maxActivationCount="50" />
                                <licenseKey sequence="4" key="8b6e7919-7107-4025-bddb-5a746e1b2691" owner="" ownerEmail="" status="InActive" lastModified="2010-02-04T15:46:02.525392+11:00" noOfActivation="0" maxActivationCount="50" />
                                <licenseKey sequence="5" key="c5cf95d8-4b5a-458b-8704-4e8e569e2e07" owner="" ownerEmail="" status="InActive" lastModified="2010-02-04T15:46:02.525392+11:00" noOfActivation="0" maxActivationCount="50" />
                </license>
                <license batchID="2" batchStatus="Active" expireDays="180" admin="WESTERN_SYDNEY\rsakhamuri" adminEmail="Ramya.Sakhamuri@site.com" requester="rsakhamuri" requesterEmail="Ramya.Sakhamuri@site.com" creationTimeStamp="2010-04-20T18:30:33.998494+10:00" activationType="OnlineOnly">
                                <note>for OTEN users 20-04-2010</note>
                                <licenseKey sequence="1" key="96af9492-0f01-4689-9443-2b098a75f08b" owner="Julie Esson" ownerEmail="ramya.sakhamuri@site.com" status="ReActivated" lastModified="2010-04-21T06:21:53.4190295Z" noOfActivation="2" maxActivationCount="50">
                                                <activator IP="172.24.108.190" userName="WESTERN_SYDNEY\rsakhamuri" email="ramya.sakhamuri@site.com" activationTimeStamp="2010-04-21T08:44:23.8095566+10:00">ramya sakhamuri</activator>
                                                <activator IP="172.24.111.148" userName="SYDNEY\JEsson" email="julie.esson@site.com" activationTimeStamp="2010-04-21T16:21:53.4190295+10:00">Julie Esson</activator>
                                </licenseKey>
                                <licenseKey sequence="2" key="9c5e56b7-8733-46df-9f2d-8b91a57fd805" owner="Kuan Sin" ownerEmail="kuan.sin@site.com" status="Active" lastModified="2010-04-21T06:33:50.4079932Z" noOfActivation="1" maxActivationCount="50">
                                                <activator IP="172.24.107.251" userName="WESTERN_SYDNEY\ksin" email="kuan.sin@site.com" activationTimeStamp="2010-04-21T16:33:50.4079932+10:00">Kuan Sin</activator>
                                </licenseKey>
                                <licenseKey sequence="3" key="a350aff9-78c0-4446-a88e-8967771211c2" owner="Carmen Vallis" ownerEmail="carmen.vallis@site.com" status="ReActivated" lastModified="2010-07-22T23:52:25.3306929Z" noOfActivation="2" maxActivationCount="50">
                                                <activator IP="172.24.108.67" userName="WESTERN_SYDNEY\cvallis2" email="carmen.vallis@site.com" activationTimeStamp="2010-04-21T16:41:42.0516367+10:00">Carmen Vallis</activator>
                                                <activator IP="172.24.111.148" userName="WESTERN_SYDNEY\cvallis2" email="carmen.vallis1@site.com" activationTimeStamp="2010-07-23T09:52:25.3306929+10:00">Carmen Vallis</activator>
                                </licenseKey>
                                <licenseKey sequence="4" key="67865ae2-036e-4be3-a601-a55d378d7694" owner="Janet Coller" ownerEmail="janet.coller@site.com" status="Active" lastModified="2010-04-21T22:21:08.9738945Z" noOfActivation="1" maxActivationCount="50">
                                                <activator IP="172.24.110.20" userName="WESTERN_SYDNEY\jcoller" email="janet.coller@site.com" activationTimeStamp="2010-04-22T08:21:08.9738945+10:00">Janet Coller</activator>
                                </licenseKey>
                                <licenseKey sequence="5" key="bd927ebb-a846-4af7-a3fe-00a827ca69fb" owner="Rob Wales" ownerEmail="robert.wales@site.com" status="Active" lastModified="2010-04-21T23:51:30.0881764Z" noOfActivation="1" maxActivationCount="50">
                                                <activator IP="172.24.109.153" userName="WESTERN_SYDNEY\rwales2" email="robert.wales@site.com" activationTimeStamp="2010-04-22T09:51:30.0881764+10:00">Rob Wales</activator>
                                </licenseKey>
                </license>
</licenseBatch>
This file can be automatically imported into MS Excel and subsequently to MS SQL Server database. It can be used for solution verification.The issue with reading such XML file using OPENXML is that not all data can be extracted in one go.  The licenseKey tag may consist of multiple activator tags however OPENXML can read only one set at the time.  The paragraph "I. Specifying the xml data type in the WITH clause" of Using OPENXML describes extracting unconsumed data into OverFlow column.The solution below required enabling BULK insert for the database user and Ad Hoc Distributed Queries.  It can be achieved by running the following set of commands:sp_addsrvrolemember @loginame = N' YourDatabaseUser', @rolename = N'bulkadmin'gosp_configure 'show advanced options',1reconfigure with overridegosp_configure 'Ad Hoc Distributed Queries',1reconfigure with overridegoSolution (combines consumed and unconsumed data into the one recordset):
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_NULLS ON
GOPRINT 'drop procedure dbo.udpLicenceKeys'
GOif exists (select * from dbo.sysobjects
 where id = object_id(N'dbo.udpLicenceKeys')
 and OBJECTPROPERTY(id, N'IsProcedure') = 1)
drop procedure dbo.udpLicenceKeys
GO

PRINT 'CREATE PROC dbo.udpLicenceKeys'
GO/******************************************************************************
   NAME:              udpLicenceKeys
   PURPOSE:   REVISIONS:   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
 ******************************************************************************/
CREATE PROCEDURE dbo.udpLicenceKeys
AS
   DECLARE  @LicenceKeys TABLE(XmlCol xml)                INSERT @LicenceKeys
                  SELECT CONVERT(xml, BulkColumn, 2) FROM
                                OPENROWSET(Bulk '\\servername\share\LicenseKeys.xml'
                                , SINGLE_BLOB) rowsetresults;                DECLARE @Data XML                SELECT @Data = XmlCol FROM @LicenceKeys                             DECLARE @idoc int
                DECLARE @doc NVARCHAR(max)
                SET @doc = (SELECT CONVERT(NVARCHAR(max),XmlCol) FROM @LicenceKeys)
                SET @doc = REPLACE(@doc,'<?xml version="1.0" encoding="utf-8"?>','')
                SET @doc = REPLACE(@doc,' xmlns="http://site.com/schemas/licensekey"','')                EXEC sp_xml_preparedocument @idoc OUTPUT, @doc                DECLARE @FindSubString VARCHAR(MAX)                SET @FindSubString = '<activator'                SELECT  T.*
                ,(CASE WHEN (LEN(convert(nvarchar(max),overflow)) -
                                                                LEN(REPLACE(convert(nvarchar(max),overflow), @FindSubString, '')))
                                                                /LEN(@FindSubString) > 0
                                                THEN REPLACE(convert(nvarchar(max),overflow)
                                                                ,'<licenseKey','<licenseKey batchID="'+CONVERT(NVARCHAR(10),BatchID)
                                                                +'" sequence="' + CONVERT(NVARCHAR(10),sequence) + '" ')
                  ELSE NULL END
                ) strXML                FROM (
                SELECT *
                FROM OPENXML (@idoc, '/licenseBatch/license/licenseKey',10)
                WITH (
                                batchID float '../@batchID'
                                ,batchStatus nvarchar(255) '../@batchStatus'
                                ,expireDays float '../@expireDays'
                                ,[admin] nvarchar(255) '../@admin'
                                ,adminEmail nvarchar(255) '../@adminEmail'
                                ,requester nvarchar(255) '../@requester'
                                ,requesterEmail nvarchar(255) '../@requesterEmail'
                                ,creationTimeStamp nvarchar(255) '../@creationTimeStamp'
                                ,activationType nvarchar(255) '../@activationType'
                                ,note nvarchar(255) '../note'
                                ,sequence float '@sequence'
                                ,[key] nvarchar(255) '@key'
                                ,[owner] nvarchar(255) '@owner'
                                ,ownerEmail nvarchar(255) '@ownerEmail'
                                ,[status] nvarchar(255) '@status'
                                ,lastModified nvarchar(255) '@lastModified'
                                ,noOfActivation float '@noOfActivation'
                                ,maxActivationCount float '@maxActivationCount'
                                ,activator nvarchar(255)
                                ,IP nvarchar(255) 'activator/@IP'
                                ,userName nvarchar(255) 'activator/@userName'
                                ,email nvarchar(255) 'activator/@email'
                                ,activationTimeStamp datetime 'activator/@activationTimeStamp'
                                ,OverFlow nvarchar(max) '@mp:xmltext')) TGO
SET QUOTED_IDENTIFIER OFF
GO
SET ANSI_NULLS ON
GO                       
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_NULLS ON
GOPRINT 'drop view dbo.udvLicenceKeys'
GOif exists (select * from dbo.sysobjects
 where id = object_id(N'dbo.udvLicenceKeys')
 and OBJECTPROPERTY(id, N'IsView') = 1)
drop view dbo.udvLicenceKeys
GOPRINT 'CREATE VIEW dbo.udvLicenceKeys'
GO/******************************************************************************
   NAME:              dbo.udvLicenceKeys
   PURPOSE:   REVISIONS:$Log$   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
 ******************************************************************************/
CREATE VIEW dbo.udvLicenceKeys
AS
SELECT  *
FROM    OPENROWSET( 'SQLNCLI'
                    ,'Server=servername;UID=YourUserName;PWD=YourPassword;'
                    ,'SET FMTONLY OFF; SET NOCOUNT ON; Execute YourDatabaseName..udpLicenceKeys')

GO
SET QUOTED_IDENTIFIER OFF
GO
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_NULLS ON
GOPRINT 'drop function dbo.udfGet_Overflow_XML'
GOif exists (select * from sysobjects
where id = object_id(N'.dbo.udfGet_Overflow_XML')
 and xtype in (N'FN', N'IF', N'TF'))
drop function dbo.udfGet_Overflow_XML
GOPRINT 'CREATE FUNCTION dbo.udfGet_Overflow_XML'
GO/******************************************************************************
   NAME:              udvGet_Overflow_XML
   PURPOSE:   REVISIONS:$Log$   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
 ******************************************************************************/
CREATE FUNCTION dbo.udfGet_Overflow_XML()
RETURNS NVARCHAR(MAX)
AS
                BEGIN
                                DECLARE  @strXML NVARCHAR(MAX)
                                                                ,@finalXML NVARCHAR(MAX)                                DECLARE crs CURSOR
                                                                FOR select strXML from dbo.udvLicenceKeys                                SET @finalXML = '<root>'                                IF @@error = 0 BEGIN
                                                OPEN crs
                                END                                FETCH NEXT FROM crs
                                INTO   @strXML                                WHILE @@FETCH_STATUS = 0 BEGIN
                                                IF @strXML IS NOT NULL BEGIN
                                                                SET @finalXML = @finalXML + @strXML
                                                END
                                                FETCH NEXT FROM crs
                                                INTO   @strXML
                                END
                                DEALLOCATE crs
                                SET @finalXML = @finalXML + '</root>'
                                RETURN (@finalXML)
                ENDGO
SET QUOTED_IDENTIFIER OFF
GO
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_NULLS ON
GOPRINT 'drop procedure dbo.udpOverflow_Recordset'
GOif exists (select * from dbo.sysobjects
 where id = object_id(N'dbo.udpOverflow_Recordset')
 and OBJECTPROPERTY(id, N'IsProcedure') = 1)
drop procedure dbo.udpOverflow_Recordset
GO

PRINT 'CREATE PROC dbo.udpOverflow_Recordset'
GO/******************************************************************************
   NAME:              udpOverflow_Recordset
   PURPOSE:   REVISIONS:   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
 ******************************************************************************/
CREATE PROCEDURE dbo.udpOverflow_Recordset
AS                DECLARE @DocHandle int
                DECLARE @XmlDocument nvarchar(MAX)                                SET @XmlDocument = (select dbo.udfGet_Overflow_XML())
                                EXEC sp_xml_preparedocument @DocHandle OUTPUT, @XmlDocument                                SELECT *
                                FROM OPENXML (@DocHandle, '/root/licenseKey/activator',1)
                                                  WITH (batchID float '../@batchID'
                                                                ,sequence float '../@sequence'
                                                                ,IP nvarchar(255) '@IP'
                                                                ,userName nvarchar(255) '@userName'
                                                                ,email nvarchar(255) '@email'
                                                                ,activationTimeStamp datetime '@activationTimeStamp'
                                                                ,activator nvarchar(255) '.')               
                                EXEC sp_xml_removedocument @DocHandleGO
SET QUOTED_IDENTIFIER OFF
GO
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_NULLS ON
GOPRINT 'drop view dbo.udvOverflow_Recordset'
GOif exists (select * from dbo.sysobjects
 where id = object_id(N'dbo.udvOverflow_Recordset')
 and OBJECTPROPERTY(id, N'IsView') = 1)
drop view dbo.udvOverflow_Recordset
GOPRINT 'CREATE VIEW dbo.udvOverflow_Recordset'
GO/******************************************************************************
   NAME:              dbo.udvOverflow_Recordset
   PURPOSE:   REVISIONS:$Log$   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
 ******************************************************************************/
CREATE VIEW dbo.udvOverflow_Recordset
AS
SELECT  *
FROM    OPENROWSET( 'SQLNCLI'
                                                                                ,'Server=servername;UID=YourUserName;PWD=YourPass;'
                    ,'SET FMTONLY OFF; SET NOCOUNT ON; Execute DatabaseName..udpOverflow_Recordset')

GO
SET QUOTED_IDENTIFIER OFF
GO
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_NULLS ON
GOPRINT 'drop view dbo.udvOverflow_Full_Recordset'
GOif exists (select * from dbo.sysobjects
 where id = object_id(N'dbo.udvOverflow_Full_Recordset')
 and OBJECTPROPERTY(id, N'IsView') = 1)
drop view dbo.udvOverflow_Full_Recordset
GOPRINT 'CREATE VIEW dbo.udvOverflow_Full_Recordset'
GO/******************************************************************************
   NAME:              dbo.udvOverflow_Full_Recordset
   PURPOSE:   REVISIONS:$Log$   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
 ******************************************************************************/
CREATE VIEW dbo.udvOverflow_Full_Recordset
AS
                SELECT DISTINCT
                                                O.batchID
                                                ,L.batchStatus
                                                ,L.expireDays
                                                ,L.admin
                                                ,L.adminEmail
                                                ,L.requester
                                                ,L.requesterEmail
                                                ,L.creationTimeStamp
                                                ,L.activationType
                                                ,L.note
                                                ,L.sequence
                                                ,L.[key]
                                                ,L.owner
                                                ,L.ownerEmail
                                                ,L.status
                                                ,L.lastModified
                                                ,L.noOfActivation
                                                ,L.maxActivationCount
                                                ,O.activator
                                                ,O.IP
                                                ,O.userName
                                                ,O.email
                                                ,O.activationTimeStamp
                                                 from dbo.udvLicenceKeys L
                INNER JOIN dbo.udvOverflow_Recordset O
                 ON L.batchID = O.batchID
                 WHERE L.sequence = O.sequenceGO
SET QUOTED_IDENTIFIER OFF
GO
SET ANSI_NULLS ON
GO
SET QUOTED_IDENTIFIER ON
GO
SET ANSI_NULLS ON
GOPRINT 'drop function dbo.udfLicenceKeys'
GOif exists (select * from sysobjects
where id = object_id(N'.dbo.udfLicenceKeys')
 and xtype in (N'FN', N'IF', N'TF'))
drop function dbo.udfLicenceKeys
GOPRINT 'CREATE FUNCTION dbo.udfLicenceKeys'
GO/******************************************************************************
   NAME:              udfLicenceKeys
   PURPOSE:   REVISIONS:$Log$   Ver        Date        Author           Description
   ---------  ----------  ---------------  ------------------------------------
 ******************************************************************************/
CREATE FUNCTION dbo.udfLicenceKeys()
RETURNS @tempTBL TABLE (
                                                                 batchID float
                                                                ,batchStatus nvarchar(255)
                                                                ,expireDays float
                                                                ,[admin] nvarchar(255)
                                                                ,adminEmail nvarchar(255)
                                                                ,requester nvarchar(255)
                                                                ,requesterEmail nvarchar(255)
                                                                ,creationTimeStamp nvarchar(255)
                                                                ,activationType nvarchar(255)
                                                                ,note nvarchar(255)
                                                                ,sequence float
                                                                ,[key] nvarchar(255)
                                                                ,[owner] nvarchar(255)
                                                                ,ownerEmail nvarchar(255)
                                                                ,[status] nvarchar(255)
                                                                ,lastModified nvarchar(255)
                                                                ,noOfActivation float
                                                                ,maxActivationCount float
                                                                ,activator nvarchar(255)
                                                                ,IP nvarchar(255)
                                                                ,userName nvarchar(255)
                                                                ,email nvarchar(255)
                                                                ,activationTimeStamp datetime)
AS
                BEGIN
                                INSERT INTO @tempTBL
                                SELECT batchID
                                                ,batchStatus
                                                ,expireDays
                                                ,admin
                                                ,adminEmail
                                                ,requester
                                                ,requesterEmail
                                                ,creationTimeStamp
                                                ,activationType
                                                ,note
                                                ,sequence
                                                ,[key]
                                                ,owner
                                                ,ownerEmail
                                                ,status
                                                ,lastModified
                                                ,noOfActivation
                                                ,maxActivationCount
                                                ,activator
                                                ,IP
                                                ,userName
                                                ,email
                                                ,activationTimeStamp
                                FROM  dbo.udvLicenceKeys                                INSERT INTO @tempTBL
                                SELECT batchID
                                                ,batchStatus
                                                ,expireDays
                                                ,admin
                                                ,adminEmail
                                                ,requester
                                                ,requesterEmail
                                                ,creationTimeStamp
                                                ,activationType
                                                ,note
                                                ,sequence
                                                ,[key]
                                                ,owner
                                                ,ownerEmail
                                                ,status
                                                ,lastModified
                                                ,noOfActivation
                                                ,maxActivationCount
                                                ,activator
                                                ,IP
                                                ,userName
                                                ,email
                                                ,activationTimeStamp
                                FROM   dbo.udvOverflow_Full_Recordset
                                RETURN
                ENDGO
SET QUOTED_IDENTIFIER OFF
GO
SET ANSI_NULLS ON
GO

 The final query:SELECT * FROM dbo.udfLicenceKeys()
 
If you find this script useful please donate generously.